lens-content-processor 0.3.0 → 0.5.0

package/dist/flattener/index.js

@@ -1,51 +1,218 @@
-import { checkTierViolation } from '../validator/tier.js';
-import { parseModule, parsePageSegments } from '../parser/module.js';
-import { parseLearningOutcome } from '../parser/learning-outcome.js';
-import { parseLens } from '../parser/lens.js';
-import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion } from '../parser/wikilink.js';
-import { parseFrontmatter } from '../parser/frontmatter.js';
-import { fileNameToSlug } from '../utils/slug.js';
-import { extractArticleExcerpt } from '../bundler/article.js';
-import { extractVideoExcerpt } from '../bundler/video.js';
+import { resolveTextLinks, enrichCardLinks } from "./resolve-text-links.js";
+import { checkTierViolation } from "../validator/tier.js";
+import { parseModule, hasFieldBeforeSegmentHeaders, } from "../parser/module.js";
+import { parseLearningOutcome, } from "../parser/learning-outcome.js";
+import { parseLens, } from "../parser/lens.js";
+import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, } from "../parser/wikilink.js";
+import { parseFrontmatter } from "../parser/frontmatter.js";
+import { fileNameToSlug } from "../utils/slug.js";
+import { extractArticleExcerpt, bundleArticleWithCollapsed, } from "../bundler/article.js";
+import { extractVideoExcerpt } from "../bundler/video.js";
+function countWords(text) {
+    return text.trim().split(/\s+/).filter(Boolean).length;
+}
 /**
- * Extract YouTube video ID from a YouTube URL.
- *
- * Supported formats:
- * - https://www.youtube.com/watch?v=VIDEO_ID
- * - https://youtube.com/watch?v=VIDEO_ID
- * - https://youtu.be/VIDEO_ID
- * - https://www.youtube.com/embed/VIDEO_ID
+ * A lens that is hidden (hide:: true) but not optional is a contradiction:
+ * users never see it, yet it counts as required to finish the section, so the
+ * section can't be completed. Flag it so editors either make it optional or
+ * unhide it.
+ */
+function checkHideOptionalConflict(optional, hide, file, line) {
+    if (hide && !optional) {
+        return {
+            file,
+            line,
+            message: "Lens is hidden (hide:: true) but required (optional is not true)",
+            suggestion: "A hidden lens cannot be completed by users — set optional:: true, or remove hide:: true to show it",
+            severity: "error",
+        };
+    }
+    return null;
+}
+/** Compute word count and video duration from a section's segments. */
+function computeSectionStats(segments) {
+    let words = 0;
+    let videoSeconds = 0;
+    for (const seg of segments) {
+        if (seg.type === "text" || seg.type === "article") {
+            words += countWords(seg.content);
+        }
+        else if (seg.type === "video") {
+            if (seg.to != null)
+                videoSeconds += seg.to - seg.from;
+        }
+    }
+    const stats = {};
+    if (words > 0)
+        stats.wordCount = words;
+    if (videoSeconds > 0)
+        stats.videoDurationSeconds = videoSeconds;
+    return stats;
+}
+/** Derive display type from segment content for UI icons/labels. */
+function computeDisplayType(segments) {
+    let hasArticle = false;
+    let hasVideo = false;
+    for (const seg of segments) {
+        if (seg.type === "article")
+            hasArticle = true;
+        if (seg.type === "video")
+            hasVideo = true;
+    }
+    if (hasArticle && hasVideo)
+        return "lens-mixed";
+    if (hasArticle)
+        return "lens-article";
+    if (hasVideo)
+        return "lens-video";
+    return undefined;
+}
+/**
+ * Extract YouTube video metadata from a URL. Returns the video ID and whether
+ * the URL is a YouTube Short (`/shorts/<id>`), so callers can pick a 9:16 layout.
+ */
+function extractVideoFromUrl(url) {
+    const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/|youtube\.com\/(shorts)\/)([a-zA-Z0-9_-]+)/);
+    if (!match)
+        return null;
+    return { videoId: match[2], isShort: match[1] === "shorts" };
+}
+export function isBoundary(item) {
+    return "__boundary" in item && item.__boundary === true;
+}
+function toKebabCase(str) {
+    return str
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/(^-|-$)/g, "");
+}
+export function validateBoundaries(items, file) {
+    const errors = [];
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries)
+        return errors;
+    // Check: sections before first boundary
+    const firstBoundaryIdx = items.findIndex(isBoundary);
+    const sectionsBefore = items
+        .slice(0, firstBoundaryIdx)
+        .filter((i) => !isBoundary(i));
+    if (sectionsBefore.length > 0) {
+        errors.push({
+            file,
+            message: "Content found outside submodule boundaries — all content must be inside a submodule",
+            severity: "error",
+        });
+    }
+    // Check: consecutive boundaries and trailing boundary
+    for (let i = 0; i < items.length; i++) {
+        if (isBoundary(items[i])) {
+            // Find next non-boundary item
+            let nextNonBoundary = i + 1;
+            while (nextNonBoundary < items.length &&
+                isBoundary(items[nextNonBoundary])) {
+                nextNonBoundary++;
+            }
+            // If this boundary has no sections before next boundary or end
+            if (nextNonBoundary > i + 1 || nextNonBoundary >= items.length) {
+                if (nextNonBoundary > i + 1) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content before next submodule`,
+                        severity: "error",
+                    });
+                }
+                else if (nextNonBoundary >= items.length) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content after this marker`,
+                        severity: "error",
+                    });
+                }
+            }
+        }
+    }
+    return errors;
+}
+export function splitAtBoundaries(items, parentSlug, parentTitle) {
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries) {
+        // No split — return single group without parentSlug
+        return [
+            {
+                slug: parentSlug,
+                title: parentTitle,
+                parentSlug: undefined,
+                parentTitle: undefined,
+                sections: items.filter((i) => !isBoundary(i)),
+                contentId: null,
+            },
+        ];
+    }
+    const groups = [];
+    let currentGroup = null;
+    for (const item of items) {
+        if (isBoundary(item)) {
+            if (currentGroup) {
+                groups.push(currentGroup);
+            }
+            const slug = item.customSlug ?? toKebabCase(item.title);
+            currentGroup = {
+                slug: `${parentSlug}/${slug}`,
+                title: item.title,
+                parentSlug,
+                parentTitle,
+                sections: [],
+                contentId: null,
+            };
+        }
+        else if (currentGroup) {
+            currentGroup.sections.push(item);
+        }
+        // sections before first boundary are orphans — handled by validateBoundaries
+    }
+    if (currentGroup) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
+/**
+ * Flatten a single ParsedLens into output Segments.
+ * This is the shared core used by all lens-processing functions.
  *
- * @returns Video ID string, or null if URL format is not recognized
+ * For each segment in the lens, calls convertSegment to resolve article/video content.
+ * Then applies collapsed content for article segments from the same source.
  */
-function extractVideoIdFromUrl(url) {
-    const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]+)/);
-    return match ? match[1] : null;
+function flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap) {
+    const errors = [];
+    const segments = [];
+    for (const parsedSegment of lens.segments) {
+        const segmentResult = convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap);
+        errors.push(...segmentResult.errors);
+        if (segmentResult.segment) {
+            segments.push(segmentResult.segment);
+        }
+    }
+    // Apply collapsed content for article segments
+    applyCollapsedContent(segments, lens.segments, lensPath, files);
+    return { segments, errors };
 }
+// ─── Module flattening ───
 /**
  * Flatten a module by resolving all references to Learning Outcomes, Lenses, and content.
- *
- * This function:
- * 1. Parses the module file
- * 2. For each Learning Outcome section, resolves the LO file
- * 3. For each Lens in the LO, resolves the lens file
- * 4. For each segment, extracts content from articles/videos as needed
- *
- * @param modulePath - Path to the module file within the files Map
- * @param files - Map of all file paths to their content
- * @param visitedPaths - Optional set of already-visited paths for cycle detection
- * @returns Flattened module with resolved sections and segments, plus any errors
  */
 export function flattenModule(modulePath, files, visitedPaths = new Set(), tierMap) {
     // Check for circular reference
     if (visitedPaths.has(modulePath)) {
         return {
             module: null,
-            errors: [{
+            modules: [],
+            errors: [
+                {
                     file: modulePath,
                     message: `Circular reference detected: ${modulePath}`,
-                    severity: 'error',
-                }],
+                    severity: "error",
+                },
+            ],
         };
     }
     visitedPaths.add(modulePath);
@@ -57,66 +224,236 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
         errors.push({
             file: modulePath,
             message: `Module file not found: ${modulePath}`,
-            severity: 'error',
+            severity: "error",
         });
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     // Parse the module
     const moduleResult = parseModule(moduleContent, modulePath);
     errors.push(...moduleResult.errors);
     if (!moduleResult.module) {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const parsedModule = moduleResult.module;
-    const flattenedSections = [];
-    // Process each section in the module
-    for (const section of parsedModule.sections) {
-        if (section.type === 'learning-outcome') {
-            // Resolve the Learning Outcome reference
-            // Create a copy of visitedPaths for this section's reference chain
-            // This allows the same file to be referenced in different sections
-            // while still detecting cycles within a single chain
+    const flatItems = [];
+    // Helper: process a section (LO or Lens) and return Section[]
+    function processSection(section, _subsectionLevel) {
+        if (section.type === "learning-outcome") {
             const sectionVisitedPaths = new Set(visitedPaths);
             const result = flattenLearningOutcomeSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
-        }
-        else if (section.type === 'page') {
-            // Page sections don't have LO references, they have inline content
-            // Parse the section body for ## Text subsections
-            const textResult = parsePageSegments(section.body, modulePath, section.line);
-            errors.push(...textResult.errors);
-            const pageSection = {
-                type: 'page',
-                meta: { title: section.title },
-                segments: textResult.segments,
-                optional: section.fields.optional?.toLowerCase() === 'true',
-                contentId: section.fields.id ?? null,
-                learningOutcomeId: null,
-                learningOutcomeName: null,
-                videoId: null,
-            };
-            flattenedSections.push(pageSection);
+            return result.sections;
         }
-        else if (section.type === 'uncategorized') {
-            // Uncategorized sections can contain ## Lens: references
-            // Each lens becomes its own section (lens-video, lens-article, or page)
+        else if (section.type === "lens") {
+            // Referenced lens (has source::) or inline lens
             const sectionVisitedPaths = new Set(visitedPaths);
-            const result = flattenUncategorizedSection(section, modulePath, files, sectionVisitedPaths, tierMap);
+            const result = flattenLensSection(section, modulePath, files, sectionVisitedPaths, tierMap, section.inlineLens);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
+            return result.sections;
         }
+        return [];
     }
-    const flattenedModule = {
-        slug: parsedModule.slug,
-        title: parsedModule.title,
+    // Process each section in the module
+    for (const section of parsedModule.sections) {
+        if (section.type === "submodule") {
+            // Emit boundary marker
+            flatItems.push({
+                __boundary: true,
+                title: section.title,
+                customSlug: section.fields.slug,
+            });
+            // Process children
+            if (section.children) {
+                for (const child of section.children) {
+                    const childSections = processSection(child, child.level + 1);
+                    flatItems.push(...childSections);
+                }
+            }
+        }
+        else {
+            // Check if this is an LO with submodules
+            if (section.type === "learning-outcome") {
+                const loSections = processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors);
+                if (loSections) {
+                    flatItems.push(...loSections);
+                    continue;
+                }
+            }
+            const sections = processSection(section);
+            flatItems.push(...sections);
+        }
+    }
+    // Validate and split at boundaries
+    const boundaryErrors = validateBoundaries(flatItems, modulePath);
+    errors.push(...boundaryErrors);
+    const groups = splitAtBoundaries(flatItems, parsedModule.slug, parsedModule.title);
+    const resultModules = groups.map((g) => ({
+        slug: g.slug,
+        title: g.title,
         contentId: parsedModule.contentId,
-        sections: flattenedSections,
+        sections: g.sections,
+        sourcePath: modulePath,
+        ...(g.parentSlug
+            ? { parentSlug: g.parentSlug, parentTitle: g.parentTitle }
+            : {}),
+        ...(moduleError ? { error: moduleError } : {}),
+        ...("addToAiContext" in parsedModule
+            ? { addToAiContext: parsedModule.addToAiContext }
+            : {}),
+    }));
+    // Enrich ::card links in text segments with computed metadata (duration, attribution, displayType)
+    for (const mod of resultModules) {
+        enrichCardLinks(mod.sections);
+    }
+    // Backwards compat: module is the first result or a merged single module
+    const primaryModule = resultModules.length > 0 ? resultModules[0] : null;
+    return {
+        module: resultModules.length === 1 && !resultModules[0].parentSlug
+            ? resultModules[0]
+            : primaryModule,
+        modules: resultModules,
+        errors,
+    };
+}
+/**
+ * Build a test Section from a ParsedTestRef with inline segments.
+ * Shared by both processLOWithSubmodules() and flattenLearningOutcomeSection().
+ */
+function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, tierMap, errors) {
+    if (!testRef.segments.length)
+        return null;
+    const testSegments = [];
+    for (const parsedSegment of testRef.segments) {
+        const segmentResult = convertSegment(parsedSegment, loPath, files, new Set(visitedPaths), tierMap);
+        errors.push(...segmentResult.errors);
+        if (segmentResult.segment) {
+            testSegments.push(segmentResult.segment);
+        }
+    }
+    if (testSegments.length === 0)
+        return null;
+    const hasFeedback = testSegments.some((s) => s.type === "question" && s.feedback);
+    return {
+        type: "test",
+        meta: { title: "Test" },
+        segments: testSegments,
+        sourcePath: loPath,
+        optional: false,
+        ...(hasFeedback && { feedback: true }),
+        contentId: testRef.id,
+        learningOutcomeId: loId,
+        learningOutcomeName: loName,
     };
-    if (moduleError) {
-        flattenedModule.error = moduleError;
+}
+/**
+ * Check if a Learning Outcome section resolves to an LO with submodules.
+ * If so, emit boundary markers + sections. Returns null if no submodules.
+ */
+function processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors) {
+    // Resolve the LO file to check for submodules
+    const source = section.fields.source;
+    if (!source)
+        return null;
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error)
+        return null;
+    const loPathResolved = resolveWikilinkPath(wikilink.path, modulePath);
+    const loPath = findFileWithExtension(loPathResolved, files);
+    if (!loPath)
+        return null;
+    const loContent = files.get(loPath);
+    if (!loContent)
+        return null;
+    const loResult = parseLearningOutcome(loContent, loPath);
+    if (!loResult.learningOutcome?.submodules)
+        return null;
+    // This LO has submodules — emit boundaries and resolve lenses
+    const items = [];
+    for (const sub of loResult.learningOutcome.submodules) {
+        items.push({
+            __boundary: true,
+            title: sub.title,
+            customSlug: sub.customSlug,
+        });
+        // Resolve each lens in this submodule group
+        for (const lensRef of sub.lenses) {
+            const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
+            if (!lensPath) {
+                const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
+                const suggestion = formatSuggestion(similarFiles, loPath) ??
+                    "Check the file path in the wiki-link";
+                errors.push({
+                    file: loPath,
+                    message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
+                    suggestion,
+                    severity: "error",
+                });
+                continue;
+            }
+            // Check tier violation
+            if (tierMap) {
+                const parentTier = tierMap.get(loPath) ?? "production";
+                const childTier = tierMap.get(lensPath) ?? "production";
+                const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, "lens");
+                if (violation) {
+                    errors.push(violation);
+                }
+                if (childTier === "ignored")
+                    continue;
+            }
+            if (visitedPaths.has(lensPath)) {
+                errors.push({
+                    file: loPath,
+                    message: `Circular reference detected: ${lensPath}`,
+                    severity: "error",
+                });
+                continue;
+            }
+            const sectionVisitedPaths = new Set(visitedPaths);
+            sectionVisitedPaths.add(lensPath);
+            const lensContent = files.get(lensPath);
+            const lensResult = parseLens(lensContent, lensPath);
+            errors.push(...lensResult.errors);
+            if (!lensResult.lens)
+                continue;
+            const lens = lensResult.lens;
+            const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, sectionVisitedPaths, tierMap);
+            errors.push(...flattenErrors);
+            const lo = loResult.learningOutcome;
+            const effectiveOptional = section.fields.optional?.toLowerCase() === "true" || lensRef.optional;
+            const effectiveHide = section.fields.hide?.toLowerCase() === "true" || lensRef.hide;
+            const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, loPath, section.line);
+            if (hideConflict)
+                errors.push(hideConflict);
+            items.push({
+                type: "lens",
+                meta: { title: lens.title || section.title },
+                segments,
+                sourcePath: lensPath,
+                optional: effectiveOptional,
+                ...(effectiveHide && { hide: true }),
+                learningOutcomeId: lo.id ?? null,
+                learningOutcomeName: loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null,
+                contentId: lens.id ?? null,
+                tldr: lens.tldr,
+                summaryForTutor: lens.summaryForTutor,
+                addToAiContext: lens.addToAiContext,
+                addToAiContextLo: lo.addToAiContext,
+                ...computeSectionStats(segments),
+                displayType: computeDisplayType(segments),
+            });
+        }
+        // Process test section if present in this submodule
+        if (sub.test) {
+            const lo = loResult.learningOutcome;
+            const testSection = buildTestSection(sub.test, lo.id ?? null, loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null, loPath, files, visitedPaths, tierMap, errors);
+            if (testSection)
+                items.push(testSection);
+        }
     }
-    return { module: flattenedModule, errors };
+    // Also emit LO parse errors
+    errors.push(...loResult.errors);
+    return items;
 }
 /**
  * Flatten a Learning Outcome section by resolving its LO file and all referenced lenses.
@@ -131,9 +468,9 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
         errors.push({
             file: modulePath,
             line: section.line,
-            message: 'Learning Outcome section missing source:: field',
+            message: "Learning Outcome section missing source:: field",
             suggestion: "Add 'source:: [[../Learning Outcomes/filename.md|Display]]'",
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -142,7 +479,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     if (!wikilink || wikilink.error) {
         const suggestion = wikilink?.correctedPath
             ? `Did you mean '[[${wikilink.correctedPath}]]'?`
-            : 'Use format [[../Learning Outcomes/filename.md|Display Text]]';
+            : "Use format [[../Learning Outcomes/filename.md|Display Text]]";
         errors.push({
             file: modulePath,
             line: section.line,
@@ -150,7 +487,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
                 ? `${wikilink.error}: ${source}`
                 : `Invalid wikilink format: ${source}`,
             suggestion,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -158,28 +495,27 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     const loPath = findFileWithExtension(loPathResolved, files);
     // Get the LO file content
     if (!loPath) {
-        // Find similar files to suggest
-        const similarFiles = findSimilarFiles(loPathResolved, files, 'Learning Outcomes');
-        const suggestion = formatSuggestion(similarFiles, modulePath) ?? 'Check the file path in the wiki-link';
+        const similarFiles = findSimilarFiles(loPathResolved, files, "Learning Outcomes");
+        const suggestion = formatSuggestion(similarFiles, modulePath) ??
+            "Check the file path in the wiki-link";
         errors.push({
             file: modulePath,
             line: section.line,
             message: `Referenced file not found: ${loPathResolved}`,
             suggestion,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
-    // Check tier violation (module → LO)
+    // Check tier violation (module -> LO)
     if (tierMap) {
-        const parentTier = tierMap.get(modulePath) ?? 'production';
-        const childTier = tierMap.get(loPath) ?? 'production';
-        const violation = checkTierViolation(modulePath, parentTier, loPath, childTier, 'learning outcome', section.line);
+        const parentTier = tierMap.get(modulePath) ?? "production";
+        const childTier = tierMap.get(loPath) ?? "production";
+        const violation = checkTierViolation(modulePath, parentTier, loPath, childTier, "learning outcome", section.line);
         if (violation) {
             errors.push(violation);
         }
-        // Skip ignored children silently (they're not processed)
-        if (childTier === 'ignored') {
+        if (childTier === "ignored") {
             return { sections: [], errors };
         }
     }
@@ -189,7 +525,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             file: modulePath,
             line: section.line,
             message: `Circular reference detected: ${loPath}`,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -206,26 +542,26 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     for (const lensRef of lo.lenses) {
         const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
         if (!lensPath) {
-            // Find similar files to suggest
-            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
-            const suggestion = formatSuggestion(similarFiles, loPath) ?? 'Check the file path in the wiki-link';
+            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
+            const suggestion = formatSuggestion(similarFiles, loPath) ??
+                "Check the file path in the wiki-link";
             errors.push({
                 file: loPath,
                 message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
                 suggestion,
-                severity: 'error',
+                severity: "error",
             });
             continue;
         }
-        // Check tier violation (LO → Lens)
+        // Check tier violation (LO -> Lens)
         if (tierMap) {
-            const parentTier = tierMap.get(loPath) ?? 'production';
-            const childTier = tierMap.get(lensPath) ?? 'production';
-            const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+            const parentTier = tierMap.get(loPath) ?? "production";
+            const childTier = tierMap.get(lensPath) ?? "production";
+            const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, "lens");
             if (violation) {
                 errors.push(violation);
             }
-            if (childTier === 'ignored') {
+            if (childTier === "ignored") {
                 continue;
             }
         }
@@ -234,7 +570,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             errors.push({
                 file: loPath,
                 message: `Circular reference detected: ${lensPath}`,
-                severity: 'error',
+                severity: "error",
             });
             continue;
         }
@@ -247,425 +583,263 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             continue;
         }
         const lens = lensResult.lens;
-        // Each lens becomes its own section
-        let sectionType = 'page';
-        const meta = { title: section.title };
-        const segments = [];
-        let videoId;
-        // Process each section in the lens
-        for (const lensSection of lens.sections) {
-            // Determine section type from lens section
-            if (lensSection.type === 'lens-article') {
-                sectionType = 'lens-article';
-                // Extract article metadata from the article file's frontmatter
-                if (lensSection.source) {
-                    const articleWikilink = parseWikilink(lensSection.source);
-                    if (articleWikilink && !articleWikilink.error) {
-                        const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                        const articlePath = findFileWithExtension(articlePathResolved, files);
-                        if (articlePath) {
-                            const articleContent = files.get(articlePath);
-                            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                            // Extract metadata fields
-                            if (articleFrontmatter.frontmatter.title) {
-                                meta.title = articleFrontmatter.frontmatter.title;
-                            }
-                            if (articleFrontmatter.frontmatter.author) {
-                                const raw = articleFrontmatter.frontmatter.author;
-                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
-                            }
-                            if (articleFrontmatter.frontmatter.source_url) {
-                                meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                            }
-                        }
-                    }
-                }
-            }
-            else if (lensSection.type === 'page') {
-                sectionType = 'page';
-                // For page sections, use the title from the ### Page: header
-                if (lensSection.title) {
-                    meta.title = lensSection.title;
-                }
-            }
-            else if (lensSection.type === 'lens-video') {
-                sectionType = 'lens-video';
-                // Extract video metadata from the video transcript file's frontmatter
-                if (lensSection.source) {
-                    const videoWikilink = parseWikilink(lensSection.source);
-                    if (videoWikilink && !videoWikilink.error) {
-                        const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                        const videoPath = findFileWithExtension(videoPathResolved, files);
-                        if (videoPath) {
-                            const videoContent = files.get(videoPath);
-                            const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                            // Extract metadata fields
-                            if (videoFrontmatter.frontmatter.title) {
-                                meta.title = videoFrontmatter.frontmatter.title;
-                            }
-                            if (videoFrontmatter.frontmatter.channel) {
-                                meta.channel = videoFrontmatter.frontmatter.channel;
-                            }
-                            // Extract video ID from YouTube URL
-                            if (videoFrontmatter.frontmatter.url) {
-                                const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                                if (extractedVideoId) {
-                                    videoId = extractedVideoId;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            // Process segments
-            for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-                errors.push(...segmentResult.errors);
-                if (segmentResult.segment) {
-                    segments.push(segmentResult.segment);
-                }
-            }
-        }
+        // Flatten lens segments using shared helper
+        const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+        errors.push(...flattenErrors);
         // Create a section for this lens
-        // Optional can come from either:
-        // 1. The LO reference in the module (section.fields.optional) - makes ALL lenses optional
-        // 2. The individual lens reference in the LO (lensRef.optional) - makes just this lens optional
+        const effectiveOptional = section.fields.optional?.toLowerCase() === "true" || lensRef.optional;
+        const effectiveHide = section.fields.hide?.toLowerCase() === "true" || lensRef.hide;
+        const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, loPath, section.line);
+        if (hideConflict)
+            errors.push(hideConflict);
         const resultSection = {
-            type: sectionType,
-            meta,
+            type: "lens",
+            meta: { title: lens.title || section.title },
             segments,
-            optional: section.fields.optional?.toLowerCase() === 'true' || lensRef.optional,
+            sourcePath: lensPath,
+            optional: effectiveOptional,
+            ...(effectiveHide && { hide: true }),
             learningOutcomeId: lo.id ?? null,
-            learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
+            learningOutcomeName: loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null,
             contentId: lens.id ?? null,
-            videoId: videoId ?? null,
+            tldr: lens.tldr,
+            summaryForTutor: lens.summaryForTutor,
+            addToAiContext: lens.addToAiContext,
+            addToAiContextLo: lo.addToAiContext,
+            ...computeSectionStats(segments),
+            displayType: computeDisplayType(segments),
         };
         sections.push(resultSection);
     }
     // After lens processing, add test section if present with inline segments
-    if (lo.test && lo.test.segments.length > 0) {
-        const testSegments = [];
-        for (const parsedSegment of lo.test.segments) {
-            // For test sections with question/chat/text segments, no source file resolution needed
-            // Create a minimal stub lensSection since these segment types don't need it
-            const stubLensSection = {
-                type: 'page',
-                title: 'Test',
-                segments: [],
-                line: 0,
-            };
-            const segmentResult = convertSegment(parsedSegment, stubLensSection, loPath, files, visitedPaths, tierMap);
-            errors.push(...segmentResult.errors);
-            if (segmentResult.segment) {
-                testSegments.push(segmentResult.segment);
-            }
-        }
-        if (testSegments.length > 0) {
-            const hasFeedback = testSegments.some((s) => s.type === 'question' && s.feedback);
-            sections.push({
-                type: 'test',
-                meta: { title: 'Test' },
-                segments: testSegments,
-                optional: false,
-                ...(hasFeedback && { feedback: true }),
-                contentId: null,
-                learningOutcomeId: lo.id ?? null,
-                learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
-                videoId: null,
-            });
-        }
+    if (lo.test) {
+        const testSection = buildTestSection(lo.test, lo.id ?? null, loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null, loPath, files, visitedPaths, tierMap, errors);
+        if (testSection)
+            sections.push(testSection);
     }
     return { sections, errors };
 }
 /**
- * Flatten an Uncategorized section by parsing its ## Lens: references.
- * Each lens becomes its own section with the appropriate type (lens-video, lens-article, or page).
+ * Flatten a # Lens: section from a module file.
+ * Handles both referenced lenses (with source:: wikilink) and inline lenses (with id:: + segments).
  */
-function flattenUncategorizedSection(section, modulePath, files, visitedPaths, tierMap) {
+function flattenLensSection(section, modulePath, files, visitedPaths, tierMap, inlineLens) {
     const errors = [];
     const sections = [];
-    // Parse the section body for ## Lens: subsections
-    const lensRefs = parseUncategorizedLensRefs(section.body, modulePath);
-    // If no lens refs found, warn and return empty array
-    if (lensRefs.length === 0) {
+    // Inline lens: has id:: + segments (no source:: at section level)
+    if (inlineLens) {
+        const { segments, errors: flattenErrors } = flattenSingleLens(inlineLens, modulePath, files, visitedPaths, tierMap);
+        errors.push(...flattenErrors);
+        const effectiveOptional = section.fields.optional?.toLowerCase() === "true";
+        const effectiveHide = section.fields.hide?.toLowerCase() === "true";
+        const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, modulePath, section.line);
+        if (hideConflict)
+            errors.push(hideConflict);
+        const resultSection = {
+            type: "lens",
+            meta: { title: inlineLens.title || section.title },
+            segments,
+            sourcePath: modulePath,
+            optional: effectiveOptional,
+            ...(effectiveHide && { hide: true }),
+            learningOutcomeId: null,
+            learningOutcomeName: null,
+            contentId: inlineLens.id ?? null,
+            tldr: inlineLens.tldr,
+            summaryForTutor: inlineLens.summaryForTutor,
+            addToAiContext: inlineLens.addToAiContext,
+            ...computeSectionStats(segments),
+            displayType: computeDisplayType(segments),
+        };
+        sections.push(resultSection);
+        return { sections, errors };
+    }
+    // Referenced lens: has source:: at section level (not from segment fields)
+    const hasSectionSource = hasFieldBeforeSegmentHeaders(section.body, "source", section.level);
+    const source = hasSectionSource ? section.fields.source : undefined;
+    if (!source) {
         errors.push({
             file: modulePath,
             line: section.line,
-            message: 'Uncategorized section has no ## Lens: references — this section will produce no output',
-            suggestion: "Add '## Lens: [[../Lenses/lens-name.md|Display]]' references",
-            severity: 'warning',
+            message: "Lens section missing source:: field (or id:: for inline lens)",
+            suggestion: "Add 'source:: [[../Lenses/filename.md|Display]]' or 'id:: <uuid>' with #### segments",
+            severity: "error",
         });
         return { sections: [], errors };
     }
-    // Process each lens reference as a separate section
-    for (const lensRef of lensRefs) {
-        const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
-        if (!lensPath) {
-            // Find similar files to suggest
-            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
-            const suggestion = formatSuggestion(similarFiles, modulePath) ?? 'Check the file path in the wiki-link';
-            errors.push({
-                file: modulePath,
-                message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
-                suggestion,
-                severity: 'error',
-            });
-            continue;
-        }
-        // Check tier violation (Uncategorized/Module → Lens)
-        if (tierMap) {
-            const parentTier = tierMap.get(modulePath) ?? 'production';
-            const childTier = tierMap.get(lensPath) ?? 'production';
-            const violation = checkTierViolation(modulePath, parentTier, lensPath, childTier, 'lens');
-            if (violation) {
-                errors.push(violation);
-            }
-            if (childTier === 'ignored') {
-                continue;
-            }
-        }
-        // Check for circular reference
-        if (visitedPaths.has(lensPath)) {
-            errors.push({
-                file: modulePath,
-                message: `Circular reference detected: ${lensPath}`,
-                severity: 'error',
-            });
-            continue;
-        }
-        visitedPaths.add(lensPath);
-        const lensContent = files.get(lensPath);
-        // Parse the lens
-        const lensResult = parseLens(lensContent, lensPath);
-        errors.push(...lensResult.errors);
-        if (!lensResult.lens) {
-            continue;
+    // Parse and resolve the wikilink
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error) {
+        const suggestion = wikilink?.correctedPath
+            ? `Did you mean '[[${wikilink.correctedPath}]]'?`
+            : "Use format [[../Lenses/filename.md|Display Text]]";
+        errors.push({
+            file: modulePath,
+            line: section.line,
+            message: wikilink?.error
+                ? `${wikilink.error}: ${source}`
+                : `Invalid wikilink format: ${source}`,
+            suggestion,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    const lensPathResolved = resolveWikilinkPath(wikilink.path, modulePath);
+    const lensPath = findFileWithExtension(lensPathResolved, files);
+    if (!lensPath) {
+        const similarFiles = findSimilarFiles(lensPathResolved, files, "Lenses");
+        const suggestion = formatSuggestion(similarFiles, modulePath) ??
+            "Check the file path in the wiki-link";
+        errors.push({
+            file: modulePath,
+            message: `Referenced lens file not found: ${lensPathResolved}`,
+            suggestion,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    // Check tier violation
+    if (tierMap) {
+        const parentTier = tierMap.get(modulePath) ?? "production";
+        const childTier = tierMap.get(lensPath) ?? "production";
+        const violation = checkTierViolation(modulePath, parentTier, lensPath, childTier, "lens");
+        if (violation) {
+            errors.push(violation);
         }
-        const lens = lensResult.lens;
-        // Each lens becomes its own section
-        let sectionType = 'page';
-        const meta = { title: section.title };
-        const segments = [];
-        let videoId;
-        // Process each section in the lens
-        for (const lensSection of lens.sections) {
-            // Determine section type from lens section
-            if (lensSection.type === 'lens-article') {
-                sectionType = 'lens-article';
-                // Extract article metadata from the article file's frontmatter
-                if (lensSection.source) {
-                    const articleWikilink = parseWikilink(lensSection.source);
-                    if (articleWikilink && !articleWikilink.error) {
-                        const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                        const articlePath = findFileWithExtension(articlePathResolved, files);
-                        if (articlePath) {
-                            const articleContent = files.get(articlePath);
-                            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                            // Extract metadata fields
-                            if (articleFrontmatter.frontmatter.title) {
-                                meta.title = articleFrontmatter.frontmatter.title;
-                            }
-                            if (articleFrontmatter.frontmatter.author) {
-                                const raw = articleFrontmatter.frontmatter.author;
-                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
-                            }
-                            if (articleFrontmatter.frontmatter.source_url) {
-                                meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                            }
-                        }
-                    }
-                }
-            }
-            else if (lensSection.type === 'page') {
-                sectionType = 'page';
-                // For page sections, use the title from the ### Page: header
-                if (lensSection.title) {
-                    meta.title = lensSection.title;
-                }
-            }
-            else if (lensSection.type === 'lens-video') {
-                sectionType = 'lens-video';
-                // Extract video metadata from the video transcript file's frontmatter
-                if (lensSection.source) {
-                    const videoWikilink = parseWikilink(lensSection.source);
-                    if (videoWikilink && !videoWikilink.error) {
-                        const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                        const videoPath = findFileWithExtension(videoPathResolved, files);
-                        if (videoPath) {
-                            const videoContent = files.get(videoPath);
-                            const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                            // Extract metadata fields
-                            if (videoFrontmatter.frontmatter.title) {
-                                meta.title = videoFrontmatter.frontmatter.title;
-                            }
-                            if (videoFrontmatter.frontmatter.channel) {
-                                meta.channel = videoFrontmatter.frontmatter.channel;
-                            }
-                            // Extract video ID from YouTube URL
-                            if (videoFrontmatter.frontmatter.url) {
-                                const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                                if (extractedVideoId) {
-                                    videoId = extractedVideoId;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            // Process segments
-            for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-                errors.push(...segmentResult.errors);
-                if (segmentResult.segment) {
-                    segments.push(segmentResult.segment);
-                }
-            }
+        if (childTier === "ignored") {
+            return { sections: [], errors };
         }
-        // Create a section for this lens
-        const resultSection = {
-            type: sectionType,
-            meta,
-            segments,
-            optional: lensRef.optional,
-            learningOutcomeId: null,
-            learningOutcomeName: null,
-            contentId: lens.id ?? null,
-            videoId: videoId ?? null,
-        };
-        sections.push(resultSection);
     }
+    // Check for circular reference
+    if (visitedPaths.has(lensPath)) {
+        errors.push({
+            file: modulePath,
+            message: `Circular reference detected: ${lensPath}`,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    visitedPaths.add(lensPath);
+    const lensContent = files.get(lensPath);
+    const lensResult = parseLens(lensContent, lensPath);
+    errors.push(...lensResult.errors);
+    if (!lensResult.lens) {
+        return { sections: [], errors };
+    }
+    const lens = lensResult.lens;
+    const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+    errors.push(...flattenErrors);
+    const effectiveOptional = section.fields.optional?.toLowerCase() === "true";
+    const effectiveHide = section.fields.hide?.toLowerCase() === "true";
+    const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, modulePath, section.line);
+    if (hideConflict)
+        errors.push(hideConflict);
+    const resultSection = {
+        type: "lens",
+        meta: { title: lens.title || section.title },
+        segments,
+        sourcePath: lensPath,
+        optional: effectiveOptional,
+        ...(effectiveHide && { hide: true }),
+        learningOutcomeId: null,
+        learningOutcomeName: null,
+        contentId: lens.id ?? null,
+        tldr: lens.tldr,
+        summaryForTutor: lens.summaryForTutor,
+        addToAiContext: lens.addToAiContext,
+        ...computeSectionStats(segments),
+        displayType: computeDisplayType(segments),
+    };
+    sections.push(resultSection);
     return { sections, errors };
 }
+// ─── Collapsed content and segment conversion ───
 /**
- * Parse ## Lens: subsections from an Uncategorized section's body.
- * Returns an array of lens references with resolved paths.
+ * After converting segments, apply collapsed_before/collapsed_after to article segments.
+ * Groups article segments by their resolved source path, then calls bundleArticleWithCollapsed
+ * for each group.
  */
-function parseUncategorizedLensRefs(body, parentPath) {
-    const lensRefs = [];
-    const lines = body.split('\n');
-    let inLensSection = false;
-    let currentFields = {};
-    let currentField = null;
-    let currentValue = [];
-    const LENS_HEADER_PATTERN = /^##\s+Lens:\s*(.*)$/i;
-    const FIELD_PATTERN = /^(\w+)::\s*(.*)$/;
-    for (let i = 0; i < lines.length; i++) {
-        const line = lines[i];
-        // Check for ## Lens: header
-        const lensMatch = line.match(LENS_HEADER_PATTERN);
-        if (lensMatch) {
-            // Save previous lens if we were in one
-            if (inLensSection) {
-                // First, finalize current field if we were collecting one
-                // This must happen BEFORE checking currentFields.source
-                if (currentField) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                }
-                if (currentFields.source) {
-                    const wikilink = parseWikilink(currentFields.source);
-                    if (wikilink) {
-                        const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                        lensRefs.push({
-                            source: currentFields.source,
-                            resolvedPath,
-                            optional: currentFields.optional?.toLowerCase() === 'true',
-                        });
-                    }
-                }
-            }
-            inLensSection = true;
-            currentFields = {};
-            currentField = null;
-            currentValue = [];
+function applyCollapsedContent(segments, parsedSegments, lensPath, files) {
+    // Build groups of article segments by resolved source path
+    const sourceGroups = new Map();
+    // Map output article segments back to parsed segments for source info
+    let articleOutputIdx = 0;
+    for (let i = 0; i < parsedSegments.length; i++) {
+        const ps = parsedSegments[i];
+        if (ps.type !== "article")
             continue;
+        // Find corresponding output segment
+        while (articleOutputIdx < segments.length &&
+            segments[articleOutputIdx].type !== "article") {
+            articleOutputIdx++;
         }
-        // Check for another ## header (end of Lens section)
-        if (line.match(/^##\s+\S/) && inLensSection) {
-            // Save current lens
-            if (currentField) {
-                currentFields[currentField] = currentValue.join('\n').trim();
-            }
-            if (currentFields.source) {
-                const wikilink = parseWikilink(currentFields.source);
-                if (wikilink && !wikilink.error) {
-                    const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                    lensRefs.push({
-                        source: currentFields.source,
-                        resolvedPath,
-                        optional: currentFields.optional?.toLowerCase() === 'true',
+        if (articleOutputIdx >= segments.length)
+            break;
+        if (ps.source) {
+            const wikilink = parseWikilink(ps.source);
+            if (wikilink && !wikilink.error) {
+                const resolved = resolveWikilinkPath(wikilink.path, lensPath);
+                const actualPath = findFileWithExtension(resolved, files);
+                if (actualPath) {
+                    if (!sourceGroups.has(actualPath)) {
+                        sourceGroups.set(actualPath, []);
+                    }
+                    sourceGroups.get(actualPath).push({
+                        segmentIndex: articleOutputIdx,
+                        fromAnchor: ps.fromAnchor,
+                        toAnchor: ps.toAnchor,
                     });
                 }
             }
-            inLensSection = false;
-            currentFields = {};
-            currentField = null;
-            currentValue = [];
-            continue;
-        }
-        if (inLensSection) {
-            // Parse fields
-            const fieldMatch = line.match(FIELD_PATTERN);
-            if (fieldMatch) {
-                // Save previous field
-                if (currentField) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                }
-                currentField = fieldMatch[1];
-                const inlineValue = fieldMatch[2].trim();
-                currentValue = inlineValue ? [inlineValue] : [];
-            }
-            else if (currentField) {
-                // Check if line starts a new section header
-                if (line.match(/^#/)) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                    currentField = null;
-                    currentValue = [];
-                }
-                else {
-                    currentValue.push(line);
-                }
-            }
         }
+        articleOutputIdx++;
     }
-    // Don't forget the last lens section
-    if (inLensSection) {
-        if (currentField) {
-            currentFields[currentField] = currentValue.join('\n').trim();
-        }
-        if (currentFields.source) {
-            const wikilink = parseWikilink(currentFields.source);
-            if (wikilink) {
-                const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                lensRefs.push({
-                    source: currentFields.source,
-                    resolvedPath,
-                    optional: currentFields.optional?.toLowerCase() === 'true',
-                });
+    // For each source group, apply collapsed content
+    for (const [articlePath, excerptInfos] of sourceGroups) {
+        // Only apply collapsed if at least one excerpt has anchors
+        const hasAnchors = excerptInfos.some((e) => e.fromAnchor || e.toAnchor);
+        if (!hasAnchors)
+            continue;
+        const articleContent = files.get(articlePath);
+        if (!articleContent)
+            continue;
+        const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map((e) => ({ from: e.fromAnchor, to: e.toAnchor })), articlePath);
+        for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
+            const segIdx = excerptInfos[i].segmentIndex;
+            const collapsed = collapsedResults[i];
+            const segment = segments[segIdx];
+            if (collapsed.collapsed_before) {
+                segment.collapsed_before = collapsed.collapsed_before;
+            }
+            if (collapsed.collapsed_after) {
+                segment.collapsed_after = collapsed.collapsed_after;
             }
         }
     }
-    return lensRefs;
 }
 /**
  * Convert a parsed lens segment into a final flattened segment.
- * For article-excerpt and video-excerpt, this involves extracting content from source files.
+ * For article and video, this involves extracting content from source files.
+ * Source comes from the parsed segment itself (set via inheritance in parseLens).
  */
-function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap) {
+function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
     const errors = [];
     switch (parsedSegment.type) {
-        case 'text': {
+        case "text": {
+            const resolved = resolveTextLinks(parsedSegment.content, lensPath, files);
+            errors.push(...resolved.errors);
             const segment = {
-                type: 'text',
-                content: parsedSegment.content,
+                type: "text",
+                content: resolved.content,
             };
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'chat': {
+        case "chat": {
             const segment = {
-                type: 'chat',
+                type: "chat",
             };
             if (parsedSegment.instructions) {
                 segment.instructions = parsedSegment.instructions;
@@ -681,157 +855,161 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             }
             return { segment, errors };
         }
-        case 'article-excerpt': {
-            // Need to resolve the article path from the lens section's source field
-            if (!lensSection.source) {
+        case "article": {
+            // Source comes from the parsed segment (set via inheritance in parseLens)
+            if (!parsedSegment.source) {
                 errors.push({
                     file: lensPath,
-                    message: 'Article section missing source:: field for article-excerpt',
-                    severity: 'error',
+                    message: "Article segment missing source (should have been set via inheritance)",
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            const wikilink = parseWikilink(lensSection.source);
+            const wikilink = parseWikilink(parsedSegment.source);
             if (!wikilink || wikilink.error) {
                 const suggestion = wikilink?.correctedPath
                     ? `Did you mean '[[${wikilink.correctedPath}]]'?`
                     : undefined;
                 errors.push({
                     file: lensPath,
-                    message: `Invalid wikilink in article source: ${lensSection.source}`,
+                    message: `Invalid wikilink in article source: ${parsedSegment.source}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
             const articlePath = findFileWithExtension(articlePathResolved, files);
             if (!articlePath) {
-                // Find similar files to suggest
-                const similarFiles = findSimilarFiles(articlePathResolved, files, 'articles');
-                const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check the file path in the wiki-link';
+                const similarFiles = findSimilarFiles(articlePathResolved, files, "articles");
+                const suggestion = formatSuggestion(similarFiles, lensPath) ??
+                    "Check the file path in the wiki-link";
                 errors.push({
                     file: lensPath,
                     message: `Referenced article file not found: ${articlePathResolved}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            // Check tier violation (Lens → Article)
+            // Check tier violation (Lens -> Article)
             if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? 'production';
-                const childTier = tierMap.get(articlePath) ?? 'production';
-                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, 'article');
+                const parentTier = tierMap.get(lensPath) ?? "production";
+                const childTier = tierMap.get(articlePath) ?? "production";
+                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, "article");
                 if (violation) {
                     errors.push(violation);
                 }
-                if (childTier === 'ignored') {
+                if (childTier === "ignored") {
                     return { segment: null, errors };
                 }
             }
-            // Check if the article path points back to an already-visited structural file
-            // This would indicate a circular reference (e.g., a lens source pointing back to an LO)
-            // Note: We only check, we don't add article paths to visitedPaths since
-            // multiple segments can legitimately reference the same article
             if (visitedPaths.has(articlePath)) {
                 errors.push({
                     file: lensPath,
                     message: `Circular reference detected: ${articlePath}`,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const articleContent = files.get(articlePath);
+            // Extract article metadata from frontmatter
+            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
+            const fm = articleFrontmatter.frontmatter;
             // Extract the excerpt
             const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
-                type: 'article-excerpt',
+                type: "article",
                 content: excerptResult.content,
             };
+            // Populate metadata from article frontmatter
+            if (fm.title)
+                segment.title = fm.title;
+            if (fm.author) {
+                const raw = fm.author;
+                segment.author = Array.isArray(raw) ? raw.join(", ") : String(raw);
+            }
+            if (fm.source_url)
+                segment.sourceUrl = fm.source_url;
+            if (fm.published)
+                segment.published = String(fm.published);
+            segment.sourcePath = articlePath;
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'video-excerpt': {
-            // Need to resolve the video/transcript path from the lens section's source field
-            if (!lensSection.source) {
+        case "video": {
+            // Source comes from the parsed segment (set via inheritance in parseLens)
+            if (!parsedSegment.source) {
                 errors.push({
                     file: lensPath,
-                    message: 'Video section missing source:: field for video-excerpt',
-                    severity: 'error',
+                    message: "Video segment missing source (should have been set via inheritance)",
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            const wikilink = parseWikilink(lensSection.source);
+            const wikilink = parseWikilink(parsedSegment.source);
             if (!wikilink || wikilink.error) {
                 const suggestion = wikilink?.correctedPath
                     ? `Did you mean '[[${wikilink.correctedPath}]]'?`
                     : undefined;
                 errors.push({
                     file: lensPath,
-                    message: `Invalid wikilink in video source: ${lensSection.source}`,
+                    message: `Invalid wikilink in video source: ${parsedSegment.source}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const videoPathResolved = resolveWikilinkPath(wikilink.path, lensPath);
             const videoPath = findFileWithExtension(videoPathResolved, files);
             if (!videoPath) {
-                // Find similar files to suggest
-                const similarFiles = findSimilarFiles(videoPathResolved, files, 'video_transcripts');
-                const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check the file path in the wiki-link';
+                const similarFiles = findSimilarFiles(videoPathResolved, files, "video_transcripts");
+                const suggestion = formatSuggestion(similarFiles, lensPath) ??
+                    "Check the file path in the wiki-link";
                 errors.push({
                     file: lensPath,
                     message: `Referenced video transcript file not found: ${videoPathResolved}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            // Check tier violation (Lens → Video)
+            // Check tier violation (Lens -> Video)
             if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? 'production';
-                const childTier = tierMap.get(videoPath) ?? 'production';
-                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, 'video transcript');
+                const parentTier = tierMap.get(lensPath) ?? "production";
+                const childTier = tierMap.get(videoPath) ?? "production";
+                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, "video transcript");
                 if (violation) {
                     errors.push(violation);
                 }
-                if (childTier === 'ignored') {
+                if (childTier === "ignored") {
                     return { segment: null, errors };
                 }
             }
-            // Check if the video path points back to an already-visited structural file
-            // This would indicate a circular reference
-            // Note: We only check, we don't add video paths to visitedPaths since
-            // multiple segments can legitimately reference the same video
             if (visitedPaths.has(videoPath)) {
                 errors.push({
                     file: lensPath,
                     message: `Circular reference detected: ${videoPath}`,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const transcriptContent = files.get(videoPath);
+            // Extract video metadata from frontmatter
+            const videoFrontmatter = parseFrontmatter(transcriptContent, videoPath);
+            const vfm = videoFrontmatter.frontmatter;
             // Look for corresponding .timestamps.json file
-            // e.g., video_transcripts/foo.md -> video_transcripts/foo.timestamps.json
-            const timestampsPath = videoPath.replace(/\.md$/, '.timestamps.json');
+            const timestampsPath = videoPath.replace(/\.md$/, ".timestamps.json");
             let timestamps;
-            // Debug: Log path resolution (uncomment to debug)
-            // console.log('DEBUG videoPath:', videoPath);
-            // console.log('DEBUG timestampsPath:', timestampsPath);
-            // console.log('DEBUG files has timestamps:', files.has(timestampsPath));
             if (files.has(timestampsPath)) {
                 try {
                     timestamps = JSON.parse(files.get(timestampsPath));
-                    // console.log('DEBUG loaded timestamps count:', timestamps.length);
                 }
                 catch {
                     // JSON parse error - will fall back to inline timestamps
@@ -840,23 +1018,36 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             // Extract the video excerpt
             const excerptResult = extractVideoExcerpt(transcriptContent, parsedSegment.fromTimeStr, parsedSegment.toTimeStr, videoPath, timestamps);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
-                type: 'video-excerpt',
+                type: "video",
                 from: excerptResult.from,
                 to: excerptResult.to,
                 transcript: excerptResult.transcript,
             };
+            // Populate metadata from video transcript frontmatter
+            if (vfm.title)
+                segment.title = vfm.title;
+            if (vfm.channel)
+                segment.channel = vfm.channel;
+            if (vfm.url) {
+                const extracted = extractVideoFromUrl(vfm.url);
+                if (extracted) {
+                    segment.videoId = extracted.videoId;
+                    if (extracted.isShort)
+                        segment.isShort = true;
+                }
+            }
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'question': {
+        case "question": {
             const segment = {
-                type: 'question',
+                type: "question",
                 content: parsedSegment.content,
             };
             if (parsedSegment.assessmentInstructions)
@@ -873,136 +1064,105 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 segment.feedback = true;
             return { segment, errors };
         }
+        case "roleplay": {
+            const segment = {
+                type: "roleplay",
+                id: parsedSegment.id,
+                content: parsedSegment.content,
+                aiInstructions: parsedSegment.aiInstructions,
+            };
+            if (parsedSegment.openingMessage)
+                segment.openingMessage = parsedSegment.openingMessage;
+            if (parsedSegment.assessmentInstructions)
+                segment.assessmentInstructions = parsedSegment.assessmentInstructions;
+            if (parsedSegment.userCustomizable)
+                segment.userCustomizable = true;
+            if (parsedSegment.optional)
+                segment.optional = true;
+            if (parsedSegment.feedback)
+                segment.feedback = true;
+            return { segment, errors };
+        }
         default:
             return { segment: null, errors };
     }
 }
 /**
  * Flatten a single Lens file into a FlattenedModule.
- *
- * This wraps a standalone Lens as a single-section module so it can be
- * rendered by the frontend using the existing Module.tsx component.
- *
- * The resulting module has:
- * - slug: 'lens/' + fileNameToSlug(lensPath)
- * - title: extracted from source metadata or lens header
- * - sections: exactly one section (lens-article, lens-video, or page)
- *
- * @param lensPath - Path to the lens file within the files Map
- * @param files - Map of all file paths to their content
- * @param tierMap - Optional tier map for filtering ignored content
- * @param preParsedLens - Optional pre-parsed lens to avoid re-parsing
- * @returns Flattened module with a single section, plus any errors
  */
 export function flattenLens(lensPath, files, tierMap, preParsedLens) {
     const errors = [];
     // Skip ignored lenses
-    if (tierMap?.get(lensPath) === 'ignored') {
-        return { module: null, errors };
+    if (tierMap?.get(lensPath) === "ignored") {
+        return { module: null, modules: [], errors };
     }
     const lensContent = files.get(lensPath);
     if (!lensContent) {
         errors.push({
             file: lensPath,
             message: `Lens file not found: ${lensPath}`,
-            severity: 'error',
+            severity: "error",
         });
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     // Use pre-parsed lens if provided, otherwise parse
-    const lens = preParsedLens ?? (() => {
-        const lensResult = parseLens(lensContent, lensPath);
-        errors.push(...lensResult.errors);
-        return lensResult.lens;
-    })();
+    const lens = preParsedLens ??
+        (() => {
+            const lensResult = parseLens(lensContent, lensPath);
+            errors.push(...lensResult.errors);
+            return lensResult.lens;
+        })();
     if (!lens) {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const visitedPaths = new Set([lensPath]);
-    // Process lens sections into a single flattened Section
-    let sectionType = 'page';
+    // Flatten lens segments using shared helper
+    const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+    errors.push(...flattenErrors);
+    // Derive title: frontmatter > first article/video segment > filename
     const meta = {};
-    const segments = [];
-    let videoId;
-    for (const lensSection of lens.sections) {
-        if (lensSection.type === 'lens-article') {
-            sectionType = 'lens-article';
-            // Extract article metadata from the article file's frontmatter
-            if (lensSection.source) {
-                const articleWikilink = parseWikilink(lensSection.source);
-                if (articleWikilink && !articleWikilink.error) {
-                    const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                    const articlePath = findFileWithExtension(articlePathResolved, files);
-                    if (articlePath) {
-                        const articleContent = files.get(articlePath);
-                        const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                        if (articleFrontmatter.frontmatter.title)
-                            meta.title = articleFrontmatter.frontmatter.title;
-                        if (articleFrontmatter.frontmatter.author)
-                            meta.author = articleFrontmatter.frontmatter.author;
-                        if (articleFrontmatter.frontmatter.source_url)
-                            meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                    }
-                }
+    if (lens.title) {
+        meta.title = lens.title;
+    }
+    else {
+        for (const seg of segments) {
+            if (seg.type === "article" && seg.title) {
+                meta.title = seg.title;
+                break;
             }
-        }
-        else if (lensSection.type === 'page') {
-            sectionType = 'page';
-            if (lensSection.title)
-                meta.title = lensSection.title;
-        }
-        else if (lensSection.type === 'lens-video') {
-            sectionType = 'lens-video';
-            // Extract video metadata from the video transcript file's frontmatter
-            if (lensSection.source) {
-                const videoWikilink = parseWikilink(lensSection.source);
-                if (videoWikilink && !videoWikilink.error) {
-                    const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                    const videoPath = findFileWithExtension(videoPathResolved, files);
-                    if (videoPath) {
-                        const videoContent = files.get(videoPath);
-                        const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                        if (videoFrontmatter.frontmatter.title)
-                            meta.title = videoFrontmatter.frontmatter.title;
-                        if (videoFrontmatter.frontmatter.channel)
-                            meta.channel = videoFrontmatter.frontmatter.channel;
-                        if (videoFrontmatter.frontmatter.url) {
-                            const extractedId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                            if (extractedId)
-                                videoId = extractedId;
-                        }
-                    }
-                }
+            else if (seg.type === "video" && seg.title) {
+                meta.title = seg.title;
+                break;
             }
         }
-        // Process segments
-        for (const parsedSegment of lensSection.segments) {
-            const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-            errors.push(...segmentResult.errors);
-            if (segmentResult.segment)
-                segments.push(segmentResult.segment);
+        if (!meta.title) {
+            meta.title = fileNameToSlug(lensPath)
+                .replace(/-/g, " ")
+                .replace(/\b\w/g, (c) => c.toUpperCase());
         }
     }
-    // Fallback title from filename if not extracted from source metadata
-    if (!meta.title) {
-        meta.title = fileNameToSlug(lensPath).replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
-    }
     const section = {
-        type: sectionType,
+        type: "lens",
         meta,
         segments,
+        sourcePath: lensPath,
         optional: false,
         learningOutcomeId: null,
         learningOutcomeName: null,
         contentId: lens.id ?? null,
-        videoId: videoId ?? null,
+        tldr: lens.tldr,
+        summaryForTutor: lens.summaryForTutor,
+        addToAiContext: lens.addToAiContext,
+        ...computeSectionStats(segments),
+        displayType: computeDisplayType(segments),
     };
     const flattenedModule = {
-        slug: 'lens/' + fileNameToSlug(lensPath),
+        slug: "lens/" + fileNameToSlug(lensPath),
         title: meta.title ?? fileNameToSlug(lensPath),
         contentId: lens.id ?? null,
         sections: [section],
+        sourcePath: lensPath,
     };
-    return { module: flattenedModule, errors };
+    return { module: flattenedModule, modules: [flattenedModule], errors };
 }
 //# sourceMappingURL=index.js.map