lens-content-processor 0.4.0 → 0.6.0

package/dist/flattener/index.js

@@ -1,24 +1,43 @@
-import { checkTierViolation } from '../validator/tier.js';
-import { parseModule, parsePageSegments } from '../parser/module.js';
-import { parseLearningOutcome } from '../parser/learning-outcome.js';
-import { parseLens } from '../parser/lens.js';
-import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion } from '../parser/wikilink.js';
-import { parseFrontmatter } from '../parser/frontmatter.js';
-import { fileNameToSlug } from '../utils/slug.js';
-import { extractArticleExcerpt, bundleArticleWithCollapsed } from '../bundler/article.js';
-import { extractVideoExcerpt } from '../bundler/video.js';
+import { resolveTextLinks, enrichCardLinks } from "./resolve-text-links.js";
+import { checkTierViolation } from "../validator/tier.js";
+import { parseModule, hasFieldBeforeSegmentHeaders, } from "../parser/module.js";
+import { parseLearningOutcome, } from "../parser/learning-outcome.js";
+import { parseLens, } from "../parser/lens.js";
+import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, } from "../parser/wikilink.js";
+import { parseFrontmatter } from "../parser/frontmatter.js";
+import { fileNameToSlug } from "../utils/slug.js";
+import { extractArticleExcerpt, bundleArticleWithCollapsed, } from "../bundler/article.js";
+import { extractVideoExcerpt } from "../bundler/video.js";
 function countWords(text) {
     return text.trim().split(/\s+/).filter(Boolean).length;
 }
+/**
+ * A lens that is hidden (hide:: true) but not optional is a contradiction:
+ * users never see it, yet it counts as required to finish the section, so the
+ * section can't be completed. Flag it so editors either make it optional or
+ * unhide it.
+ */
+function checkHideOptionalConflict(optional, hide, file, line) {
+    if (hide && !optional) {
+        return {
+            file,
+            line,
+            message: "Lens is hidden (hide:: true) but required (optional is not true)",
+            suggestion: "A hidden lens cannot be completed by users — set optional:: true, or remove hide:: true to show it",
+            severity: "error",
+        };
+    }
+    return null;
+}
 /** Compute word count and video duration from a section's segments. */
 function computeSectionStats(segments) {
     let words = 0;
     let videoSeconds = 0;
     for (const seg of segments) {
-        if (seg.type === 'text' || seg.type === 'article-excerpt') {
+        if (seg.type === "text" || seg.type === "article") {
             words += countWords(seg.content);
         }
-        else if (seg.type === 'video-excerpt') {
+        else if (seg.type === "video") {
             if (seg.to != null)
                 videoSeconds += seg.to - seg.from;
         }
@@ -30,26 +49,42 @@ function computeSectionStats(segments) {
         stats.videoDurationSeconds = videoSeconds;
     return stats;
 }
+/** Derive display type from segment content for UI icons/labels. */
+function computeDisplayType(segments) {
+    let hasArticle = false;
+    let hasVideo = false;
+    for (const seg of segments) {
+        if (seg.type === "article")
+            hasArticle = true;
+        if (seg.type === "video")
+            hasVideo = true;
+    }
+    if (hasArticle && hasVideo)
+        return "lens-mixed";
+    if (hasArticle)
+        return "lens-article";
+    if (hasVideo)
+        return "lens-video";
+    return undefined;
+}
 /**
- * Extract YouTube video ID from a YouTube URL.
- *
- * Supported formats:
- * - https://www.youtube.com/watch?v=VIDEO_ID
- * - https://youtube.com/watch?v=VIDEO_ID
- * - https://youtu.be/VIDEO_ID
- * - https://www.youtube.com/embed/VIDEO_ID
- *
- * @returns Video ID string, or null if URL format is not recognized
+ * Extract YouTube video metadata from a URL. Returns the video ID and whether
+ * the URL is a YouTube Short (`/shorts/<id>`), so callers can pick a 9:16 layout.
  */
-function extractVideoIdFromUrl(url) {
-    const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]+)/);
-    return match ? match[1] : null;
+function extractVideoFromUrl(url) {
+    const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/|youtube\.com\/(shorts)\/)([a-zA-Z0-9_-]+)/);
+    if (!match)
+        return null;
+    return { videoId: match[2], isShort: match[1] === "shorts" };
 }
 export function isBoundary(item) {
-    return '__boundary' in item && item.__boundary === true;
+    return "__boundary" in item && item.__boundary === true;
 }
 function toKebabCase(str) {
-    return str.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
+    return str
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/(^-|-$)/g, "");
 }
 export function validateBoundaries(items, file) {
     const errors = [];
@@ -58,12 +93,14 @@ export function validateBoundaries(items, file) {
         return errors;
     // Check: sections before first boundary
     const firstBoundaryIdx = items.findIndex(isBoundary);
-    const sectionsBefore = items.slice(0, firstBoundaryIdx).filter(i => !isBoundary(i));
+    const sectionsBefore = items
+        .slice(0, firstBoundaryIdx)
+        .filter((i) => !isBoundary(i));
     if (sectionsBefore.length > 0) {
         errors.push({
             file,
-            message: 'Content found outside submodule boundaries — all content must be inside a submodule',
-            severity: 'error',
+            message: "Content found outside submodule boundaries — all content must be inside a submodule",
+            severity: "error",
         });
     }
     // Check: consecutive boundaries and trailing boundary
@@ -71,25 +108,24 @@ export function validateBoundaries(items, file) {
         if (isBoundary(items[i])) {
             // Find next non-boundary item
             let nextNonBoundary = i + 1;
-            while (nextNonBoundary < items.length && isBoundary(items[nextNonBoundary])) {
+            while (nextNonBoundary < items.length &&
+                isBoundary(items[nextNonBoundary])) {
                 nextNonBoundary++;
             }
             // If this boundary has no sections before next boundary or end
             if (nextNonBoundary > i + 1 || nextNonBoundary >= items.length) {
-                // nextNonBoundary > i + 1 means consecutive boundaries
-                // nextNonBoundary >= items.length means trailing boundary with no content
                 if (nextNonBoundary > i + 1) {
                     errors.push({
                         file,
                         message: `Submodule "${items[i].title}" is empty — no content before next submodule`,
-                        severity: 'error',
+                        severity: "error",
                     });
                 }
                 else if (nextNonBoundary >= items.length) {
                     errors.push({
                         file,
                         message: `Submodule "${items[i].title}" is empty — no content after this marker`,
-                        severity: 'error',
+                        severity: "error",
                     });
                 }
             }
@@ -101,14 +137,16 @@ export function splitAtBoundaries(items, parentSlug, parentTitle) {
     const hasBoundaries = items.some(isBoundary);
     if (!hasBoundaries) {
         // No split — return single group without parentSlug
-        return [{
+        return [
+            {
                 slug: parentSlug,
                 title: parentTitle,
                 parentSlug: undefined,
                 parentTitle: undefined,
-                sections: items.filter(i => !isBoundary(i)),
+                sections: items.filter((i) => !isBoundary(i)),
                 contentId: null,
-            }];
+            },
+        ];
     }
     const groups = [];
     let currentGroup = null;
@@ -138,18 +176,29 @@ export function splitAtBoundaries(items, parentSlug, parentTitle) {
     return groups;
 }
 /**
- * Flatten a module by resolving all references to Learning Outcomes, Lenses, and content.
- *
- * This function:
- * 1. Parses the module file
- * 2. For each Learning Outcome section, resolves the LO file
- * 3. For each Lens in the LO, resolves the lens file
- * 4. For each segment, extracts content from articles/videos as needed
+ * Flatten a single ParsedLens into output Segments.
+ * This is the shared core used by all lens-processing functions.
  *
- * @param modulePath - Path to the module file within the files Map
- * @param files - Map of all file paths to their content
- * @param visitedPaths - Optional set of already-visited paths for cycle detection
- * @returns Flattened module with resolved sections and segments, plus any errors
+ * For each segment in the lens, calls convertSegment to resolve article/video content.
+ * Then applies collapsed content for article segments from the same source.
+ */
+function flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap) {
+    const errors = [];
+    const segments = [];
+    for (const parsedSegment of lens.segments) {
+        const segmentResult = convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap);
+        errors.push(...segmentResult.errors);
+        if (segmentResult.segment) {
+            segments.push(segmentResult.segment);
+        }
+    }
+    // Apply collapsed content for article segments
+    applyCollapsedContent(segments, lens.segments, lensPath, files);
+    return { segments, errors };
+}
+// ─── Module flattening ───
+/**
+ * Flatten a module by resolving all references to Learning Outcomes, Lenses, and content.
  */
 export function flattenModule(modulePath, files, visitedPaths = new Set(), tierMap) {
     // Check for circular reference
@@ -157,11 +206,13 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
         return {
             module: null,
             modules: [],
-            errors: [{
+            errors: [
+                {
                     file: modulePath,
                     message: `Circular reference detected: ${modulePath}`,
-                    severity: 'error',
-                }],
+                    severity: "error",
+                },
+            ],
         };
     }
     visitedPaths.add(modulePath);
@@ -173,7 +224,7 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
         errors.push({
             file: modulePath,
             message: `Module file not found: ${modulePath}`,
-            severity: 'error',
+            severity: "error",
         });
         return { module: null, modules: [], errors };
     }
@@ -185,33 +236,18 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
     }
     const parsedModule = moduleResult.module;
     const flatItems = [];
-    // Helper: process a section (LO, Page, Uncategorized) and return Section[]
-    function processSection(section, subsectionLevel) {
-        if (section.type === 'learning-outcome') {
+    // Helper: process a section (LO or Lens) and return Section[]
+    function processSection(section, _subsectionLevel) {
+        if (section.type === "learning-outcome") {
             const sectionVisitedPaths = new Set(visitedPaths);
             const result = flattenLearningOutcomeSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
             return result.sections;
         }
-        else if (section.type === 'page') {
-            const level = subsectionLevel ?? 2;
-            const textResult = parsePageSegments(section.body, modulePath, section.line, level);
-            errors.push(...textResult.errors);
-            return [{
-                    type: 'page',
-                    meta: { title: section.title },
-                    segments: textResult.segments,
-                    optional: section.fields.optional?.toLowerCase() === 'true',
-                    contentId: section.fields.id ?? null,
-                    learningOutcomeId: null,
-                    learningOutcomeName: null,
-                    videoId: null,
-                    ...computeSectionStats(textResult.segments),
-                }];
-        }
-        else if (section.type === 'uncategorized') {
+        else if (section.type === "lens") {
+            // Referenced lens (has source::) or inline lens
             const sectionVisitedPaths = new Set(visitedPaths);
-            const result = flattenUncategorizedSection(section, modulePath, files, sectionVisitedPaths, tierMap);
+            const result = flattenLensSection(section, modulePath, files, sectionVisitedPaths, tierMap, section.inlineLens);
             errors.push(...result.errors);
             return result.sections;
         }
@@ -219,14 +255,14 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
     }
     // Process each section in the module
     for (const section of parsedModule.sections) {
-        if (section.type === 'submodule') {
+        if (section.type === "submodule") {
             // Emit boundary marker
             flatItems.push({
                 __boundary: true,
                 title: section.title,
                 customSlug: section.fields.slug,
             });
-            // Process children at level+1
+            // Process children
             if (section.children) {
                 for (const child of section.children) {
                     const childSections = processSection(child, child.level + 1);
@@ -236,7 +272,7 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
         }
         else {
             // Check if this is an LO with submodules
-            if (section.type === 'learning-outcome') {
+            if (section.type === "learning-outcome") {
                 const loSections = processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors);
                 if (loSections) {
                     flatItems.push(...loSections);
@@ -251,17 +287,33 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
     const boundaryErrors = validateBoundaries(flatItems, modulePath);
     errors.push(...boundaryErrors);
     const groups = splitAtBoundaries(flatItems, parsedModule.slug, parsedModule.title);
-    const resultModules = groups.map(g => ({
+    const resultModules = groups.map((g) => ({
         slug: g.slug,
         title: g.title,
         contentId: parsedModule.contentId,
         sections: g.sections,
-        ...(g.parentSlug ? { parentSlug: g.parentSlug, parentTitle: g.parentTitle } : {}),
+        sourcePath: modulePath,
+        ...(g.parentSlug
+            ? { parentSlug: g.parentSlug, parentTitle: g.parentTitle }
+            : {}),
         ...(moduleError ? { error: moduleError } : {}),
+        ...("addToAiContext" in parsedModule
+            ? { addToAiContext: parsedModule.addToAiContext }
+            : {}),
     }));
+    // Enrich ::card links in text segments with computed metadata (duration, attribution, displayType)
+    for (const mod of resultModules) {
+        enrichCardLinks(mod.sections);
+    }
     // Backwards compat: module is the first result or a merged single module
     const primaryModule = resultModules.length > 0 ? resultModules[0] : null;
-    return { module: resultModules.length === 1 && !resultModules[0].parentSlug ? resultModules[0] : primaryModule, modules: resultModules, errors };
+    return {
+        module: resultModules.length === 1 && !resultModules[0].parentSlug
+            ? resultModules[0]
+            : primaryModule,
+        modules: resultModules,
+        errors,
+    };
 }
 /**
  * Build a test Section from a ParsedTestRef with inline segments.
@@ -271,14 +323,8 @@ function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, ti
     if (!testRef.segments.length)
         return null;
     const testSegments = [];
-    const stubLensSection = {
-        type: 'page',
-        title: 'Test',
-        segments: [],
-        line: 0,
-    };
     for (const parsedSegment of testRef.segments) {
-        const segmentResult = convertSegment(parsedSegment, stubLensSection, loPath, files, new Set(visitedPaths), tierMap);
+        const segmentResult = convertSegment(parsedSegment, loPath, files, new Set(visitedPaths), tierMap);
         errors.push(...segmentResult.errors);
         if (segmentResult.segment) {
             testSegments.push(segmentResult.segment);
@@ -286,17 +332,17 @@ function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, ti
     }
     if (testSegments.length === 0)
         return null;
-    const hasFeedback = testSegments.some((s) => s.type === 'question' && s.feedback);
+    const hasFeedback = testSegments.some((s) => s.type === "question" && s.feedback);
     return {
-        type: 'test',
-        meta: { title: 'Test' },
+        type: "test",
+        meta: { title: "Test" },
         segments: testSegments,
+        sourcePath: loPath,
         optional: false,
         ...(hasFeedback && { feedback: true }),
-        contentId: null,
+        contentId: testRef.id,
         learningOutcomeId: loId,
         learningOutcomeName: loName,
-        videoId: null,
     };
 }
 /**
@@ -333,32 +379,33 @@ function processLOWithSubmodules(section, modulePath, files, visitedPaths, tierM
         for (const lensRef of sub.lenses) {
             const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
             if (!lensPath) {
-                const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
-                const suggestion = formatSuggestion(similarFiles, loPath) ?? 'Check the file path in the wiki-link';
+                const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
+                const suggestion = formatSuggestion(similarFiles, loPath) ??
+                    "Check the file path in the wiki-link";
                 errors.push({
                     file: loPath,
                     message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 continue;
             }
             // Check tier violation
             if (tierMap) {
-                const parentTier = tierMap.get(loPath) ?? 'production';
-                const childTier = tierMap.get(lensPath) ?? 'production';
-                const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+                const parentTier = tierMap.get(loPath) ?? "production";
+                const childTier = tierMap.get(lensPath) ?? "production";
+                const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, "lens");
                 if (violation) {
                     errors.push(violation);
                 }
-                if (childTier === 'ignored')
+                if (childTier === "ignored")
                     continue;
             }
             if (visitedPaths.has(lensPath)) {
                 errors.push({
                     file: loPath,
                     message: `Circular reference detected: ${lensPath}`,
-                    severity: 'error',
+                    severity: "error",
                 });
                 continue;
             }
@@ -370,92 +417,41 @@ function processLOWithSubmodules(section, modulePath, files, visitedPaths, tierM
             if (!lensResult.lens)
                 continue;
             const lens = lensResult.lens;
-            let sectionType = 'page';
-            const meta = { title: section.title };
-            const segments = [];
-            let videoId;
-            for (const lensSection of lens.sections) {
-                if (lensSection.type === 'lens-article') {
-                    sectionType = 'lens-article';
-                    if (lensSection.source) {
-                        const articleWikilink = parseWikilink(lensSection.source);
-                        if (articleWikilink && !articleWikilink.error) {
-                            const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                            const articlePath = findFileWithExtension(articlePathResolved, files);
-                            if (articlePath) {
-                                const articleContent = files.get(articlePath);
-                                const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                                if (articleFrontmatter.frontmatter.title)
-                                    meta.title = articleFrontmatter.frontmatter.title;
-                                if (articleFrontmatter.frontmatter.author) {
-                                    const raw = articleFrontmatter.frontmatter.author;
-                                    meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
-                                }
-                                if (articleFrontmatter.frontmatter.source_url)
-                                    meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                            }
-                        }
-                    }
-                }
-                else if (lensSection.type === 'page') {
-                    sectionType = 'page';
-                    if (lensSection.title)
-                        meta.title = lensSection.title;
-                }
-                else if (lensSection.type === 'lens-video') {
-                    sectionType = 'lens-video';
-                    if (lensSection.source) {
-                        const videoWikilink = parseWikilink(lensSection.source);
-                        if (videoWikilink && !videoWikilink.error) {
-                            const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                            const videoPath = findFileWithExtension(videoPathResolved, files);
-                            if (videoPath) {
-                                const videoContent = files.get(videoPath);
-                                const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                                if (videoFrontmatter.frontmatter.title)
-                                    meta.title = videoFrontmatter.frontmatter.title;
-                                if (videoFrontmatter.frontmatter.channel)
-                                    meta.channel = videoFrontmatter.frontmatter.channel;
-                                if (videoFrontmatter.frontmatter.url) {
-                                    const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                                    if (extractedVideoId)
-                                        videoId = extractedVideoId;
-                                }
-                            }
-                        }
-                    }
-                }
-                for (const parsedSegment of lensSection.segments) {
-                    const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, sectionVisitedPaths, tierMap);
-                    errors.push(...segmentResult.errors);
-                    if (segmentResult.segment)
-                        segments.push(segmentResult.segment);
-                }
-                applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
-            }
+            const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, sectionVisitedPaths, tierMap);
+            errors.push(...flattenErrors);
             const lo = loResult.learningOutcome;
+            const effectiveOptional = section.fields.optional?.toLowerCase() === "true" || lensRef.optional;
+            const effectiveHide = section.fields.hide?.toLowerCase() === "true" || lensRef.hide;
+            const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, loPath, section.line);
+            if (hideConflict)
+                errors.push(hideConflict);
             items.push({
-                type: sectionType,
-                meta,
+                type: "lens",
+                meta: { title: lens.title || section.title },
                 segments,
-                optional: section.fields.optional?.toLowerCase() === 'true' || lensRef.optional,
+                sourcePath: lensPath,
+                optional: effectiveOptional,
+                ...(effectiveHide && { hide: true }),
                 learningOutcomeId: lo.id ?? null,
-                learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
+                learningOutcomeName: loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null,
                 contentId: lens.id ?? null,
                 tldr: lens.tldr,
-                videoId: videoId ?? null,
+                summaryForTutor: lens.summaryForTutor,
+                addToAiContext: lens.addToAiContext,
+                addToAiContextLo: lo.addToAiContext,
                 ...computeSectionStats(segments),
+                displayType: computeDisplayType(segments),
             });
         }
         // Process test section if present in this submodule
         if (sub.test) {
             const lo = loResult.learningOutcome;
-            const testSection = buildTestSection(sub.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+            const testSection = buildTestSection(sub.test, lo.id ?? null, loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null, loPath, files, visitedPaths, tierMap, errors);
             if (testSection)
                 items.push(testSection);
         }
     }
-    // Also emit LO parse errors (we skipped earlier since we returned non-null)
+    // Also emit LO parse errors
     errors.push(...loResult.errors);
     return items;
 }
@@ -472,9 +468,9 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
         errors.push({
             file: modulePath,
             line: section.line,
-            message: 'Learning Outcome section missing source:: field',
+            message: "Learning Outcome section missing source:: field",
             suggestion: "Add 'source:: [[../Learning Outcomes/filename.md|Display]]'",
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -483,7 +479,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     if (!wikilink || wikilink.error) {
         const suggestion = wikilink?.correctedPath
             ? `Did you mean '[[${wikilink.correctedPath}]]'?`
-            : 'Use format [[../Learning Outcomes/filename.md|Display Text]]';
+            : "Use format [[../Learning Outcomes/filename.md|Display Text]]";
         errors.push({
             file: modulePath,
             line: section.line,
@@ -491,7 +487,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
                 ? `${wikilink.error}: ${source}`
                 : `Invalid wikilink format: ${source}`,
             suggestion,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -499,28 +495,27 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     const loPath = findFileWithExtension(loPathResolved, files);
     // Get the LO file content
     if (!loPath) {
-        // Find similar files to suggest
-        const similarFiles = findSimilarFiles(loPathResolved, files, 'Learning Outcomes');
-        const suggestion = formatSuggestion(similarFiles, modulePath) ?? 'Check the file path in the wiki-link';
+        const similarFiles = findSimilarFiles(loPathResolved, files, "Learning Outcomes");
+        const suggestion = formatSuggestion(similarFiles, modulePath) ??
+            "Check the file path in the wiki-link";
         errors.push({
             file: modulePath,
             line: section.line,
             message: `Referenced file not found: ${loPathResolved}`,
             suggestion,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
-    // Check tier violation (module → LO)
+    // Check tier violation (module -> LO)
     if (tierMap) {
-        const parentTier = tierMap.get(modulePath) ?? 'production';
-        const childTier = tierMap.get(loPath) ?? 'production';
-        const violation = checkTierViolation(modulePath, parentTier, loPath, childTier, 'learning outcome', section.line);
+        const parentTier = tierMap.get(modulePath) ?? "production";
+        const childTier = tierMap.get(loPath) ?? "production";
+        const violation = checkTierViolation(modulePath, parentTier, loPath, childTier, "learning outcome", section.line);
         if (violation) {
             errors.push(violation);
         }
-        // Skip ignored children silently (they're not processed)
-        if (childTier === 'ignored') {
+        if (childTier === "ignored") {
             return { sections: [], errors };
         }
     }
@@ -530,7 +525,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             file: modulePath,
             line: section.line,
             message: `Circular reference detected: ${loPath}`,
-            severity: 'error',
+            severity: "error",
         });
         return { sections: [], errors };
     }
@@ -547,26 +542,26 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
     for (const lensRef of lo.lenses) {
         const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
         if (!lensPath) {
-            // Find similar files to suggest
-            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
-            const suggestion = formatSuggestion(similarFiles, loPath) ?? 'Check the file path in the wiki-link';
+            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
+            const suggestion = formatSuggestion(similarFiles, loPath) ??
+                "Check the file path in the wiki-link";
             errors.push({
                 file: loPath,
                 message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
                 suggestion,
-                severity: 'error',
+                severity: "error",
             });
             continue;
         }
-        // Check tier violation (LO → Lens)
+        // Check tier violation (LO -> Lens)
         if (tierMap) {
-            const parentTier = tierMap.get(loPath) ?? 'production';
-            const childTier = tierMap.get(lensPath) ?? 'production';
-            const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+            const parentTier = tierMap.get(loPath) ?? "production";
+            const childTier = tierMap.get(lensPath) ?? "production";
+            const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, "lens");
             if (violation) {
                 errors.push(violation);
             }
-            if (childTier === 'ignored') {
+            if (childTier === "ignored") {
                 continue;
             }
         }
@@ -575,7 +570,7 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             errors.push({
                 file: loPath,
                 message: `Circular reference detected: ${lensPath}`,
-                severity: 'error',
+                severity: "error",
             });
             continue;
         }
@@ -588,465 +583,269 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             continue;
         }
         const lens = lensResult.lens;
-        // Each lens becomes its own section
-        let sectionType = 'page';
-        const meta = { title: section.title };
-        const segments = [];
-        let videoId;
-        // Process each section in the lens
-        for (const lensSection of lens.sections) {
-            // Determine section type from lens section
-            if (lensSection.type === 'lens-article') {
-                sectionType = 'lens-article';
-                // Extract article metadata from the article file's frontmatter
-                if (lensSection.source) {
-                    const articleWikilink = parseWikilink(lensSection.source);
-                    if (articleWikilink && !articleWikilink.error) {
-                        const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                        const articlePath = findFileWithExtension(articlePathResolved, files);
-                        if (articlePath) {
-                            const articleContent = files.get(articlePath);
-                            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                            // Extract metadata fields
-                            if (articleFrontmatter.frontmatter.title) {
-                                meta.title = articleFrontmatter.frontmatter.title;
-                            }
-                            if (articleFrontmatter.frontmatter.author) {
-                                const raw = articleFrontmatter.frontmatter.author;
-                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
-                            }
-                            if (articleFrontmatter.frontmatter.source_url) {
-                                meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                            }
-                            if (articleFrontmatter.frontmatter.published) {
-                                meta.published = String(articleFrontmatter.frontmatter.published);
-                            }
-                        }
-                    }
-                }
-            }
-            else if (lensSection.type === 'page') {
-                sectionType = 'page';
-                // For page sections, use the title from the ### Page: header
-                if (lensSection.title) {
-                    meta.title = lensSection.title;
-                }
-            }
-            else if (lensSection.type === 'lens-video') {
-                sectionType = 'lens-video';
-                // Extract video metadata from the video transcript file's frontmatter
-                if (lensSection.source) {
-                    const videoWikilink = parseWikilink(lensSection.source);
-                    if (videoWikilink && !videoWikilink.error) {
-                        const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                        const videoPath = findFileWithExtension(videoPathResolved, files);
-                        if (videoPath) {
-                            const videoContent = files.get(videoPath);
-                            const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                            // Extract metadata fields
-                            if (videoFrontmatter.frontmatter.title) {
-                                meta.title = videoFrontmatter.frontmatter.title;
-                            }
-                            if (videoFrontmatter.frontmatter.channel) {
-                                meta.channel = videoFrontmatter.frontmatter.channel;
-                            }
-                            // Extract video ID from YouTube URL
-                            if (videoFrontmatter.frontmatter.url) {
-                                const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                                if (extractedVideoId) {
-                                    videoId = extractedVideoId;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            // Process segments
-            for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-                errors.push(...segmentResult.errors);
-                if (segmentResult.segment) {
-                    segments.push(segmentResult.segment);
-                }
-            }
-            // Apply collapsed content to article-excerpt segments
-            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
-        }
+        // Flatten lens segments using shared helper
+        const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+        errors.push(...flattenErrors);
         // Create a section for this lens
-        // Optional can come from either:
-        // 1. The LO reference in the module (section.fields.optional) - makes ALL lenses optional
-        // 2. The individual lens reference in the LO (lensRef.optional) - makes just this lens optional
+        const effectiveOptional = section.fields.optional?.toLowerCase() === "true" || lensRef.optional;
+        const effectiveHide = section.fields.hide?.toLowerCase() === "true" || lensRef.hide;
+        const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, loPath, section.line);
+        if (hideConflict)
+            errors.push(hideConflict);
         const resultSection = {
-            type: sectionType,
-            meta,
+            type: "lens",
+            meta: { title: lens.title || section.title },
             segments,
-            optional: section.fields.optional?.toLowerCase() === 'true' || lensRef.optional,
+            sourcePath: lensPath,
+            optional: effectiveOptional,
+            ...(effectiveHide && { hide: true }),
             learningOutcomeId: lo.id ?? null,
-            learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
+            learningOutcomeName: loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null,
             contentId: lens.id ?? null,
             tldr: lens.tldr,
-            videoId: videoId ?? null,
+            summaryForTutor: lens.summaryForTutor,
+            addToAiContext: lens.addToAiContext,
+            addToAiContextLo: lo.addToAiContext,
             ...computeSectionStats(segments),
+            displayType: computeDisplayType(segments),
         };
         sections.push(resultSection);
     }
     // After lens processing, add test section if present with inline segments
     if (lo.test) {
-        const testSection = buildTestSection(lo.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+        const testSection = buildTestSection(lo.test, lo.id ?? null, loPath.split("/").pop()?.replace(/\.md$/i, "") ?? null, loPath, files, visitedPaths, tierMap, errors);
         if (testSection)
             sections.push(testSection);
     }
     return { sections, errors };
 }
 /**
- * Flatten an Uncategorized section by parsing its ## Lens: references.
- * Each lens becomes its own section with the appropriate type (lens-video, lens-article, or page).
+ * Flatten a # Lens: section from a module file.
+ * Handles both referenced lenses (with source:: wikilink) and inline lenses (with id:: + segments).
  */
-function flattenUncategorizedSection(section, modulePath, files, visitedPaths, tierMap) {
+function flattenLensSection(section, modulePath, files, visitedPaths, tierMap, inlineLens) {
     const errors = [];
     const sections = [];
-    // Parse the section body for ## Lens: subsections
-    const lensRefs = parseUncategorizedLensRefs(section.body, modulePath);
-    // If no lens refs found, warn and return empty array
-    if (lensRefs.length === 0) {
+    // Inline lens: has id:: + segments (no source:: at section level)
+    if (inlineLens) {
+        const { segments, errors: flattenErrors } = flattenSingleLens(inlineLens, modulePath, files, visitedPaths, tierMap);
+        errors.push(...flattenErrors);
+        // optional:: / hide:: only count as lens-level when they appear BEFORE the
+        // first #### segment header. Otherwise a #### Chat (or any segment) with
+        // optional:: true would leak its field up to section.fields and wrongly
+        // mark the whole inline lens optional. (Same discipline as source:: below.)
+        const effectiveOptional = hasFieldBeforeSegmentHeaders(section.body, "optional", section.level) &&
+            section.fields.optional?.toLowerCase() === "true";
+        const effectiveHide = hasFieldBeforeSegmentHeaders(section.body, "hide", section.level) &&
+            section.fields.hide?.toLowerCase() === "true";
+        const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, modulePath, section.line);
+        if (hideConflict)
+            errors.push(hideConflict);
+        const resultSection = {
+            type: "lens",
+            meta: { title: inlineLens.title || section.title },
+            segments,
+            sourcePath: modulePath,
+            optional: effectiveOptional,
+            ...(effectiveHide && { hide: true }),
+            learningOutcomeId: null,
+            learningOutcomeName: null,
+            contentId: inlineLens.id ?? null,
+            tldr: inlineLens.tldr,
+            summaryForTutor: inlineLens.summaryForTutor,
+            addToAiContext: inlineLens.addToAiContext,
+            ...computeSectionStats(segments),
+            displayType: computeDisplayType(segments),
+        };
+        sections.push(resultSection);
+        return { sections, errors };
+    }
+    // Referenced lens: has source:: at section level (not from segment fields)
+    const hasSectionSource = hasFieldBeforeSegmentHeaders(section.body, "source", section.level);
+    const source = hasSectionSource ? section.fields.source : undefined;
+    if (!source) {
         errors.push({
             file: modulePath,
             line: section.line,
-            message: 'Uncategorized section has no ## Lens: references — this section will produce no output',
-            suggestion: "Add '## Lens: [[../Lenses/lens-name.md|Display]]' references",
-            severity: 'warning',
+            message: "Lens section missing source:: field (or id:: for inline lens)",
+            suggestion: "Add 'source:: [[../Lenses/filename.md|Display]]' or 'id:: <uuid>' with #### segments",
+            severity: "error",
         });
         return { sections: [], errors };
     }
-    // Process each lens reference as a separate section
-    for (const lensRef of lensRefs) {
-        const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
-        if (!lensPath) {
-            // Find similar files to suggest
-            const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
-            const suggestion = formatSuggestion(similarFiles, modulePath) ?? 'Check the file path in the wiki-link';
-            errors.push({
-                file: modulePath,
-                message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
-                suggestion,
-                severity: 'error',
-            });
-            continue;
-        }
-        // Check tier violation (Uncategorized/Module → Lens)
-        if (tierMap) {
-            const parentTier = tierMap.get(modulePath) ?? 'production';
-            const childTier = tierMap.get(lensPath) ?? 'production';
-            const violation = checkTierViolation(modulePath, parentTier, lensPath, childTier, 'lens');
-            if (violation) {
-                errors.push(violation);
-            }
-            if (childTier === 'ignored') {
-                continue;
-            }
-        }
-        // Check for circular reference
-        if (visitedPaths.has(lensPath)) {
-            errors.push({
-                file: modulePath,
-                message: `Circular reference detected: ${lensPath}`,
-                severity: 'error',
-            });
-            continue;
-        }
-        visitedPaths.add(lensPath);
-        const lensContent = files.get(lensPath);
-        // Parse the lens
-        const lensResult = parseLens(lensContent, lensPath);
-        errors.push(...lensResult.errors);
-        if (!lensResult.lens) {
-            continue;
+    // Parse and resolve the wikilink
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error) {
+        const suggestion = wikilink?.correctedPath
+            ? `Did you mean '[[${wikilink.correctedPath}]]'?`
+            : "Use format [[../Lenses/filename.md|Display Text]]";
+        errors.push({
+            file: modulePath,
+            line: section.line,
+            message: wikilink?.error
+                ? `${wikilink.error}: ${source}`
+                : `Invalid wikilink format: ${source}`,
+            suggestion,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    const lensPathResolved = resolveWikilinkPath(wikilink.path, modulePath);
+    const lensPath = findFileWithExtension(lensPathResolved, files);
+    if (!lensPath) {
+        const similarFiles = findSimilarFiles(lensPathResolved, files, "Lenses");
+        const suggestion = formatSuggestion(similarFiles, modulePath) ??
+            "Check the file path in the wiki-link";
+        errors.push({
+            file: modulePath,
+            message: `Referenced lens file not found: ${lensPathResolved}`,
+            suggestion,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    // Check tier violation
+    if (tierMap) {
+        const parentTier = tierMap.get(modulePath) ?? "production";
+        const childTier = tierMap.get(lensPath) ?? "production";
+        const violation = checkTierViolation(modulePath, parentTier, lensPath, childTier, "lens");
+        if (violation) {
+            errors.push(violation);
         }
-        const lens = lensResult.lens;
-        // Each lens becomes its own section
-        let sectionType = 'page';
-        const meta = { title: section.title };
-        const segments = [];
-        let videoId;
-        // Process each section in the lens
-        for (const lensSection of lens.sections) {
-            // Determine section type from lens section
-            if (lensSection.type === 'lens-article') {
-                sectionType = 'lens-article';
-                // Extract article metadata from the article file's frontmatter
-                if (lensSection.source) {
-                    const articleWikilink = parseWikilink(lensSection.source);
-                    if (articleWikilink && !articleWikilink.error) {
-                        const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                        const articlePath = findFileWithExtension(articlePathResolved, files);
-                        if (articlePath) {
-                            const articleContent = files.get(articlePath);
-                            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                            // Extract metadata fields
-                            if (articleFrontmatter.frontmatter.title) {
-                                meta.title = articleFrontmatter.frontmatter.title;
-                            }
-                            if (articleFrontmatter.frontmatter.author) {
-                                const raw = articleFrontmatter.frontmatter.author;
-                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
-                            }
-                            if (articleFrontmatter.frontmatter.source_url) {
-                                meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                            }
-                            if (articleFrontmatter.frontmatter.published) {
-                                meta.published = String(articleFrontmatter.frontmatter.published);
-                            }
-                        }
-                    }
-                }
-            }
-            else if (lensSection.type === 'page') {
-                sectionType = 'page';
-                // For page sections, use the title from the ### Page: header
-                if (lensSection.title) {
-                    meta.title = lensSection.title;
-                }
-            }
-            else if (lensSection.type === 'lens-video') {
-                sectionType = 'lens-video';
-                // Extract video metadata from the video transcript file's frontmatter
-                if (lensSection.source) {
-                    const videoWikilink = parseWikilink(lensSection.source);
-                    if (videoWikilink && !videoWikilink.error) {
-                        const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                        const videoPath = findFileWithExtension(videoPathResolved, files);
-                        if (videoPath) {
-                            const videoContent = files.get(videoPath);
-                            const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                            // Extract metadata fields
-                            if (videoFrontmatter.frontmatter.title) {
-                                meta.title = videoFrontmatter.frontmatter.title;
-                            }
-                            if (videoFrontmatter.frontmatter.channel) {
-                                meta.channel = videoFrontmatter.frontmatter.channel;
-                            }
-                            // Extract video ID from YouTube URL
-                            if (videoFrontmatter.frontmatter.url) {
-                                const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                                if (extractedVideoId) {
-                                    videoId = extractedVideoId;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            // Process segments
-            for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-                errors.push(...segmentResult.errors);
-                if (segmentResult.segment) {
-                    segments.push(segmentResult.segment);
-                }
-            }
-            // Apply collapsed content to article-excerpt segments
-            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
+        if (childTier === "ignored") {
+            return { sections: [], errors };
         }
-        // Create a section for this lens
-        const resultSection = {
-            type: sectionType,
-            meta,
-            segments,
-            optional: lensRef.optional,
-            learningOutcomeId: null,
-            learningOutcomeName: null,
-            contentId: lens.id ?? null,
-            tldr: lens.tldr,
-            videoId: videoId ?? null,
-            ...computeSectionStats(segments),
-        };
-        sections.push(resultSection);
     }
+    // Check for circular reference
+    if (visitedPaths.has(lensPath)) {
+        errors.push({
+            file: modulePath,
+            message: `Circular reference detected: ${lensPath}`,
+            severity: "error",
+        });
+        return { sections: [], errors };
+    }
+    visitedPaths.add(lensPath);
+    const lensContent = files.get(lensPath);
+    const lensResult = parseLens(lensContent, lensPath);
+    errors.push(...lensResult.errors);
+    if (!lensResult.lens) {
+        return { sections: [], errors };
+    }
+    const lens = lensResult.lens;
+    const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+    errors.push(...flattenErrors);
+    const effectiveOptional = section.fields.optional?.toLowerCase() === "true";
+    const effectiveHide = section.fields.hide?.toLowerCase() === "true";
+    const hideConflict = checkHideOptionalConflict(effectiveOptional, effectiveHide, modulePath, section.line);
+    if (hideConflict)
+        errors.push(hideConflict);
+    const resultSection = {
+        type: "lens",
+        meta: { title: lens.title || section.title },
+        segments,
+        sourcePath: lensPath,
+        optional: effectiveOptional,
+        ...(effectiveHide && { hide: true }),
+        learningOutcomeId: null,
+        learningOutcomeName: null,
+        contentId: lens.id ?? null,
+        tldr: lens.tldr,
+        summaryForTutor: lens.summaryForTutor,
+        addToAiContext: lens.addToAiContext,
+        ...computeSectionStats(segments),
+        displayType: computeDisplayType(segments),
+    };
+    sections.push(resultSection);
     return { sections, errors };
 }
+// ─── Collapsed content and segment conversion ───
 /**
- * Parse ## Lens: subsections from an Uncategorized section's body.
- * Returns an array of lens references with resolved paths.
+ * After converting segments, apply collapsed_before/collapsed_after to article segments.
+ * Groups article segments by their resolved source path, then calls bundleArticleWithCollapsed
+ * for each group.
  */
-function parseUncategorizedLensRefs(body, parentPath) {
-    const lensRefs = [];
-    const lines = body.split('\n');
-    let inLensSection = false;
-    let currentFields = {};
-    let currentField = null;
-    let currentValue = [];
-    const LENS_HEADER_PATTERN = /^##\s+Lens:\s*(.*)$/i;
-    const FIELD_PATTERN = /^(\w+)::\s*(.*)$/;
-    for (let i = 0; i < lines.length; i++) {
-        const line = lines[i];
-        // Check for ## Lens: header
-        const lensMatch = line.match(LENS_HEADER_PATTERN);
-        if (lensMatch) {
-            // Save previous lens if we were in one
-            if (inLensSection) {
-                // First, finalize current field if we were collecting one
-                // This must happen BEFORE checking currentFields.source
-                if (currentField) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                }
-                if (currentFields.source) {
-                    const wikilink = parseWikilink(currentFields.source);
-                    if (wikilink) {
-                        const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                        lensRefs.push({
-                            source: currentFields.source,
-                            resolvedPath,
-                            optional: currentFields.optional?.toLowerCase() === 'true',
-                        });
-                    }
-                }
-            }
-            inLensSection = true;
-            currentFields = {};
-            currentField = null;
-            currentValue = [];
+function applyCollapsedContent(segments, parsedSegments, lensPath, files) {
+    // Build groups of article segments by resolved source path
+    const sourceGroups = new Map();
+    // Map output article segments back to parsed segments for source info
+    let articleOutputIdx = 0;
+    for (let i = 0; i < parsedSegments.length; i++) {
+        const ps = parsedSegments[i];
+        if (ps.type !== "article")
             continue;
+        // Find corresponding output segment
+        while (articleOutputIdx < segments.length &&
+            segments[articleOutputIdx].type !== "article") {
+            articleOutputIdx++;
         }
-        // Check for another ## header (end of Lens section)
-        if (line.match(/^##\s+\S/) && inLensSection) {
-            // Save current lens
-            if (currentField) {
-                currentFields[currentField] = currentValue.join('\n').trim();
-            }
-            if (currentFields.source) {
-                const wikilink = parseWikilink(currentFields.source);
-                if (wikilink && !wikilink.error) {
-                    const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                    lensRefs.push({
-                        source: currentFields.source,
-                        resolvedPath,
-                        optional: currentFields.optional?.toLowerCase() === 'true',
+        if (articleOutputIdx >= segments.length)
+            break;
+        if (ps.source) {
+            const wikilink = parseWikilink(ps.source);
+            if (wikilink && !wikilink.error) {
+                const resolved = resolveWikilinkPath(wikilink.path, lensPath);
+                const actualPath = findFileWithExtension(resolved, files);
+                if (actualPath) {
+                    if (!sourceGroups.has(actualPath)) {
+                        sourceGroups.set(actualPath, []);
+                    }
+                    sourceGroups.get(actualPath).push({
+                        segmentIndex: articleOutputIdx,
+                        fromAnchor: ps.fromAnchor,
+                        toAnchor: ps.toAnchor,
                     });
                 }
             }
-            inLensSection = false;
-            currentFields = {};
-            currentField = null;
-            currentValue = [];
-            continue;
-        }
-        if (inLensSection) {
-            // Parse fields
-            const fieldMatch = line.match(FIELD_PATTERN);
-            if (fieldMatch) {
-                // Save previous field
-                if (currentField) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                }
-                currentField = fieldMatch[1];
-                const inlineValue = fieldMatch[2].trim();
-                currentValue = inlineValue ? [inlineValue] : [];
-            }
-            else if (currentField) {
-                // Check if line starts a new section header
-                if (line.match(/^#/)) {
-                    currentFields[currentField] = currentValue.join('\n').trim();
-                    currentField = null;
-                    currentValue = [];
-                }
-                else {
-                    currentValue.push(line);
-                }
-            }
         }
+        articleOutputIdx++;
     }
-    // Don't forget the last lens section
-    if (inLensSection) {
-        if (currentField) {
-            currentFields[currentField] = currentValue.join('\n').trim();
-        }
-        if (currentFields.source) {
-            const wikilink = parseWikilink(currentFields.source);
-            if (wikilink) {
-                const resolvedPath = resolveWikilinkPath(wikilink.path, parentPath);
-                lensRefs.push({
-                    source: currentFields.source,
-                    resolvedPath,
-                    optional: currentFields.optional?.toLowerCase() === 'true',
-                });
+    // For each source group, apply collapsed content
+    for (const [articlePath, excerptInfos] of sourceGroups) {
+        // Only apply collapsed if at least one excerpt has anchors
+        const hasAnchors = excerptInfos.some((e) => e.fromAnchor || e.toAnchor);
+        if (!hasAnchors)
+            continue;
+        const articleContent = files.get(articlePath);
+        if (!articleContent)
+            continue;
+        const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map((e) => ({ from: e.fromAnchor, to: e.toAnchor })), articlePath);
+        for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
+            const segIdx = excerptInfos[i].segmentIndex;
+            const collapsed = collapsedResults[i];
+            const segment = segments[segIdx];
+            if (collapsed.collapsed_before) {
+                segment.collapsed_before = collapsed.collapsed_before;
+            }
+            if (collapsed.collapsed_after) {
+                segment.collapsed_after = collapsed.collapsed_after;
             }
-        }
-    }
-    return lensRefs;
-}
-/**
- * After converting segments, apply collapsed_before/collapsed_after to article-excerpt segments.
- * This resolves the article source once and calls bundleArticleWithCollapsed to compute
- * which parts of the article are outside the excerpted ranges.
- */
-function applyCollapsedContent(segments, parsedSegments, lensSection, lensPath, files) {
-    // Only process lens-article sections with article-excerpt segments
-    if (lensSection.type !== 'lens-article' || !lensSection.source)
-        return;
-    // Collect article-excerpt segment indices
-    const excerptIndices = [];
-    for (let i = 0; i < segments.length; i++) {
-        if (segments[i].type === 'article-excerpt') {
-            excerptIndices.push(i);
-        }
-    }
-    if (excerptIndices.length === 0)
-        return;
-    // Map to parsed segments to get anchors (order-preserving)
-    const parsedExcerpts = parsedSegments.filter(ps => ps.type === 'article-excerpt');
-    const excerptInfos = excerptIndices.map((segIdx, i) => ({
-        index: segIdx,
-        from: parsedExcerpts[i]?.fromAnchor,
-        to: parsedExcerpts[i]?.toAnchor,
-    }));
-    // Only apply collapsed if at least one excerpt has anchors
-    const hasAnchors = excerptInfos.some(e => e.from || e.to);
-    if (!hasAnchors)
-        return;
-    // Resolve article path
-    const wikilink = parseWikilink(lensSection.source);
-    if (!wikilink || wikilink.error)
-        return;
-    const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
-    const articlePath = findFileWithExtension(articlePathResolved, files);
-    if (!articlePath)
-        return;
-    const articleContent = files.get(articlePath);
-    // Call bundleArticleWithCollapsed
-    const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map(e => ({ from: e.from, to: e.to })), articlePath);
-    // Overlay collapsed_before/collapsed_after onto segments
-    for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
-        const segIdx = excerptInfos[i].index;
-        const collapsed = collapsedResults[i];
-        const segment = segments[segIdx];
-        if (collapsed.collapsed_before) {
-            segment.collapsed_before = collapsed.collapsed_before;
-        }
-        if (collapsed.collapsed_after) {
-            segment.collapsed_after = collapsed.collapsed_after;
         }
     }
 }
 /**
  * Convert a parsed lens segment into a final flattened segment.
- * For article-excerpt and video-excerpt, this involves extracting content from source files.
+ * For article and video, this involves extracting content from source files.
+ * Source comes from the parsed segment itself (set via inheritance in parseLens).
  */
-function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap) {
+function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
     const errors = [];
     switch (parsedSegment.type) {
-        case 'text': {
+        case "text": {
+            const resolved = resolveTextLinks(parsedSegment.content, lensPath, files);
+            errors.push(...resolved.errors);
             const segment = {
-                type: 'text',
-                content: parsedSegment.content,
+                type: "text",
+                content: resolved.content,
             };
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'chat': {
+        case "chat": {
             const segment = {
-                type: 'chat',
+                type: "chat",
             };
             if (parsedSegment.instructions) {
                 segment.instructions = parsedSegment.instructions;
@@ -1057,73 +856,72 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             if (parsedSegment.hidePreviousContentFromTutor) {
                 segment.hidePreviousContentFromTutor = true;
             }
-            if (parsedSegment.optional) {
-                segment.optional = true;
+            if (parsedSegment.minMessages !== undefined) {
+                segment.minMessages = parsedSegment.minMessages;
             }
             return { segment, errors };
         }
-        case 'article-excerpt': {
-            // Need to resolve the article path from the lens section's source field
-            if (!lensSection.source) {
+        case "article": {
+            // Source comes from the parsed segment (set via inheritance in parseLens)
+            if (!parsedSegment.source) {
                 errors.push({
                     file: lensPath,
-                    message: 'Article section missing source:: field for article-excerpt',
-                    severity: 'error',
+                    message: "Article segment missing source (should have been set via inheritance)",
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            const wikilink = parseWikilink(lensSection.source);
+            const wikilink = parseWikilink(parsedSegment.source);
             if (!wikilink || wikilink.error) {
                 const suggestion = wikilink?.correctedPath
                     ? `Did you mean '[[${wikilink.correctedPath}]]'?`
                     : undefined;
                 errors.push({
                     file: lensPath,
-                    message: `Invalid wikilink in article source: ${lensSection.source}`,
+                    message: `Invalid wikilink in article source: ${parsedSegment.source}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
             const articlePath = findFileWithExtension(articlePathResolved, files);
             if (!articlePath) {
-                // Find similar files to suggest
-                const similarFiles = findSimilarFiles(articlePathResolved, files, 'articles');
-                const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check the file path in the wiki-link';
+                const similarFiles = findSimilarFiles(articlePathResolved, files, "articles");
+                const suggestion = formatSuggestion(similarFiles, lensPath) ??
+                    "Check the file path in the wiki-link";
                 errors.push({
                     file: lensPath,
                     message: `Referenced article file not found: ${articlePathResolved}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            // Check tier violation (Lens → Article)
+            // Check tier violation (Lens -> Article)
             if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? 'production';
-                const childTier = tierMap.get(articlePath) ?? 'production';
-                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, 'article');
+                const parentTier = tierMap.get(lensPath) ?? "production";
+                const childTier = tierMap.get(articlePath) ?? "production";
+                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, "article");
                 if (violation) {
                     errors.push(violation);
                 }
-                if (childTier === 'ignored') {
+                if (childTier === "ignored") {
                     return { segment: null, errors };
                 }
             }
-            // Check if the article path points back to an already-visited structural file
-            // This would indicate a circular reference (e.g., a lens source pointing back to an LO)
-            // Note: We only check, we don't add article paths to visitedPaths since
-            // multiple segments can legitimately reference the same article
             if (visitedPaths.has(articlePath)) {
                 errors.push({
                     file: lensPath,
                     message: `Circular reference detected: ${articlePath}`,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const articleContent = files.get(articlePath);
+            // Extract article metadata from frontmatter
+            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
+            const fm = articleFrontmatter.frontmatter;
             // Extract the excerpt
             const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
             if (excerptResult.error) {
@@ -1131,88 +929,93 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 return { segment: null, errors };
             }
             const segment = {
-                type: 'article-excerpt',
+                type: "article",
                 content: excerptResult.content,
             };
+            // Populate metadata from article frontmatter
+            if (fm.title)
+                segment.title = fm.title;
+            if (fm.author) {
+                const raw = fm.author;
+                segment.author = Array.isArray(raw) ? raw.join(", ") : String(raw);
+            }
+            if (fm.source_url)
+                segment.sourceUrl = fm.source_url;
+            if (fm.published)
+                segment.published = String(fm.published);
+            segment.sourcePath = articlePath;
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'video-excerpt': {
-            // Need to resolve the video/transcript path from the lens section's source field
-            if (!lensSection.source) {
+        case "video": {
+            // Source comes from the parsed segment (set via inheritance in parseLens)
+            if (!parsedSegment.source) {
                 errors.push({
                     file: lensPath,
-                    message: 'Video section missing source:: field for video-excerpt',
-                    severity: 'error',
+                    message: "Video segment missing source (should have been set via inheritance)",
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            const wikilink = parseWikilink(lensSection.source);
+            const wikilink = parseWikilink(parsedSegment.source);
             if (!wikilink || wikilink.error) {
                 const suggestion = wikilink?.correctedPath
                     ? `Did you mean '[[${wikilink.correctedPath}]]'?`
                     : undefined;
                 errors.push({
                     file: lensPath,
-                    message: `Invalid wikilink in video source: ${lensSection.source}`,
+                    message: `Invalid wikilink in video source: ${parsedSegment.source}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const videoPathResolved = resolveWikilinkPath(wikilink.path, lensPath);
             const videoPath = findFileWithExtension(videoPathResolved, files);
             if (!videoPath) {
-                // Find similar files to suggest
-                const similarFiles = findSimilarFiles(videoPathResolved, files, 'video_transcripts');
-                const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check the file path in the wiki-link';
+                const similarFiles = findSimilarFiles(videoPathResolved, files, "video_transcripts");
+                const suggestion = formatSuggestion(similarFiles, lensPath) ??
+                    "Check the file path in the wiki-link";
                 errors.push({
                     file: lensPath,
                     message: `Referenced video transcript file not found: ${videoPathResolved}`,
                     suggestion,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
-            // Check tier violation (Lens → Video)
+            // Check tier violation (Lens -> Video)
             if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? 'production';
-                const childTier = tierMap.get(videoPath) ?? 'production';
-                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, 'video transcript');
+                const parentTier = tierMap.get(lensPath) ?? "production";
+                const childTier = tierMap.get(videoPath) ?? "production";
+                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, "video transcript");
                 if (violation) {
                     errors.push(violation);
                 }
-                if (childTier === 'ignored') {
+                if (childTier === "ignored") {
                     return { segment: null, errors };
                 }
             }
-            // Check if the video path points back to an already-visited structural file
-            // This would indicate a circular reference
-            // Note: We only check, we don't add video paths to visitedPaths since
-            // multiple segments can legitimately reference the same video
             if (visitedPaths.has(videoPath)) {
                 errors.push({
                     file: lensPath,
                     message: `Circular reference detected: ${videoPath}`,
-                    severity: 'error',
+                    severity: "error",
                 });
                 return { segment: null, errors };
             }
             const transcriptContent = files.get(videoPath);
+            // Extract video metadata from frontmatter
+            const videoFrontmatter = parseFrontmatter(transcriptContent, videoPath);
+            const vfm = videoFrontmatter.frontmatter;
             // Look for corresponding .timestamps.json file
-            // e.g., video_transcripts/foo.md -> video_transcripts/foo.timestamps.json
-            const timestampsPath = videoPath.replace(/\.md$/, '.timestamps.json');
+            const timestampsPath = videoPath.replace(/\.md$/, ".timestamps.json");
             let timestamps;
-            // Debug: Log path resolution (uncomment to debug)
-            // console.log('DEBUG videoPath:', videoPath);
-            // console.log('DEBUG timestampsPath:', timestampsPath);
-            // console.log('DEBUG files has timestamps:', files.has(timestampsPath));
             if (files.has(timestampsPath)) {
                 try {
                     timestamps = JSON.parse(files.get(timestampsPath));
-                    // console.log('DEBUG loaded timestamps count:', timestamps.length);
                 }
                 catch {
                     // JSON parse error - will fall back to inline timestamps
@@ -1225,19 +1028,32 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 return { segment: null, errors };
             }
             const segment = {
-                type: 'video-excerpt',
+                type: "video",
                 from: excerptResult.from,
                 to: excerptResult.to,
                 transcript: excerptResult.transcript,
             };
+            // Populate metadata from video transcript frontmatter
+            if (vfm.title)
+                segment.title = vfm.title;
+            if (vfm.channel)
+                segment.channel = vfm.channel;
+            if (vfm.url) {
+                const extracted = extractVideoFromUrl(vfm.url);
+                if (extracted) {
+                    segment.videoId = extracted.videoId;
+                    if (extracted.isShort)
+                        segment.isShort = true;
+                }
+            }
             if (parsedSegment.optional) {
                 segment.optional = true;
             }
             return { segment, errors };
         }
-        case 'question': {
+        case "question": {
             const segment = {
-                type: 'question',
+                type: "question",
                 content: parsedSegment.content,
             };
             if (parsedSegment.assessmentInstructions)
@@ -1248,15 +1064,15 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 segment.maxChars = parsedSegment.maxChars;
             if (parsedSegment.enforceVoice)
                 segment.enforceVoice = true;
-            if (parsedSegment.optional)
-                segment.optional = true;
             if (parsedSegment.feedback)
                 segment.feedback = true;
+            if (parsedSegment.minMessages !== undefined)
+                segment.minMessages = parsedSegment.minMessages;
             return { segment, errors };
         }
-        case 'roleplay': {
+        case "roleplay": {
             const segment = {
-                type: 'roleplay',
+                type: "roleplay",
                 id: parsedSegment.id,
                 content: parsedSegment.content,
                 aiInstructions: parsedSegment.aiInstructions,
@@ -1265,6 +1081,8 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 segment.openingMessage = parsedSegment.openingMessage;
             if (parsedSegment.assessmentInstructions)
                 segment.assessmentInstructions = parsedSegment.assessmentInstructions;
+            if (parsedSegment.userCustomizable)
+                segment.userCustomizable = true;
             if (parsedSegment.optional)
                 segment.optional = true;
             if (parsedSegment.feedback)
@@ -1277,25 +1095,11 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
 }
 /**
  * Flatten a single Lens file into a FlattenedModule.
- *
- * This wraps a standalone Lens as a single-section module so it can be
- * rendered by the frontend using the existing Module.tsx component.
- *
- * The resulting module has:
- * - slug: 'lens/' + fileNameToSlug(lensPath)
- * - title: extracted from source metadata or lens header
- * - sections: exactly one section (lens-article, lens-video, or page)
- *
- * @param lensPath - Path to the lens file within the files Map
- * @param files - Map of all file paths to their content
- * @param tierMap - Optional tier map for filtering ignored content
- * @param preParsedLens - Optional pre-parsed lens to avoid re-parsing
- * @returns Flattened module with a single section, plus any errors
  */
 export function flattenLens(lensPath, files, tierMap, preParsedLens) {
     const errors = [];
     // Skip ignored lenses
-    if (tierMap?.get(lensPath) === 'ignored') {
+    if (tierMap?.get(lensPath) === "ignored") {
         return { module: null, modules: [], errors };
     }
     const lensContent = files.get(lensPath);
@@ -1303,109 +1107,67 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
         errors.push({
             file: lensPath,
             message: `Lens file not found: ${lensPath}`,
-            severity: 'error',
+            severity: "error",
         });
         return { module: null, modules: [], errors };
     }
     // Use pre-parsed lens if provided, otherwise parse
-    const lens = preParsedLens ?? (() => {
-        const lensResult = parseLens(lensContent, lensPath);
-        errors.push(...lensResult.errors);
-        return lensResult.lens;
-    })();
+    const lens = preParsedLens ??
+        (() => {
+            const lensResult = parseLens(lensContent, lensPath);
+            errors.push(...lensResult.errors);
+            return lensResult.lens;
+        })();
     if (!lens) {
         return { module: null, modules: [], errors };
     }
     const visitedPaths = new Set([lensPath]);
-    // Process lens sections into a single flattened Section
-    let sectionType = 'page';
+    // Flatten lens segments using shared helper
+    const { segments, errors: flattenErrors } = flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap);
+    errors.push(...flattenErrors);
+    // Derive title: frontmatter > first article/video segment > filename
     const meta = {};
-    const segments = [];
-    let videoId;
-    for (const lensSection of lens.sections) {
-        if (lensSection.type === 'lens-article') {
-            sectionType = 'lens-article';
-            // Extract article metadata from the article file's frontmatter
-            if (lensSection.source) {
-                const articleWikilink = parseWikilink(lensSection.source);
-                if (articleWikilink && !articleWikilink.error) {
-                    const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
-                    const articlePath = findFileWithExtension(articlePathResolved, files);
-                    if (articlePath) {
-                        const articleContent = files.get(articlePath);
-                        const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-                        if (articleFrontmatter.frontmatter.title)
-                            meta.title = articleFrontmatter.frontmatter.title;
-                        if (articleFrontmatter.frontmatter.author)
-                            meta.author = articleFrontmatter.frontmatter.author;
-                        if (articleFrontmatter.frontmatter.source_url)
-                            meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
-                        if (articleFrontmatter.frontmatter.published)
-                            meta.published = String(articleFrontmatter.frontmatter.published);
-                    }
-                }
+    if (lens.title) {
+        meta.title = lens.title;
+    }
+    else {
+        for (const seg of segments) {
+            if (seg.type === "article" && seg.title) {
+                meta.title = seg.title;
+                break;
             }
-        }
-        else if (lensSection.type === 'page') {
-            sectionType = 'page';
-            if (lensSection.title)
-                meta.title = lensSection.title;
-        }
-        else if (lensSection.type === 'lens-video') {
-            sectionType = 'lens-video';
-            // Extract video metadata from the video transcript file's frontmatter
-            if (lensSection.source) {
-                const videoWikilink = parseWikilink(lensSection.source);
-                if (videoWikilink && !videoWikilink.error) {
-                    const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
-                    const videoPath = findFileWithExtension(videoPathResolved, files);
-                    if (videoPath) {
-                        const videoContent = files.get(videoPath);
-                        const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
-                        if (videoFrontmatter.frontmatter.title)
-                            meta.title = videoFrontmatter.frontmatter.title;
-                        if (videoFrontmatter.frontmatter.channel)
-                            meta.channel = videoFrontmatter.frontmatter.channel;
-                        if (videoFrontmatter.frontmatter.url) {
-                            const extractedId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
-                            if (extractedId)
-                                videoId = extractedId;
-                        }
-                    }
-                }
+            else if (seg.type === "video" && seg.title) {
+                meta.title = seg.title;
+                break;
             }
         }
-        // Process segments
-        for (const parsedSegment of lensSection.segments) {
-            const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
-            errors.push(...segmentResult.errors);
-            if (segmentResult.segment)
-                segments.push(segmentResult.segment);
+        if (!meta.title) {
+            meta.title = fileNameToSlug(lensPath)
+                .replace(/-/g, " ")
+                .replace(/\b\w/g, (c) => c.toUpperCase());
         }
-        // Apply collapsed content to article-excerpt segments
-        applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
-    }
-    // Fallback title from filename if not extracted from source metadata
-    if (!meta.title) {
-        meta.title = fileNameToSlug(lensPath).replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
     }
     const section = {
-        type: sectionType,
+        type: "lens",
         meta,
         segments,
+        sourcePath: lensPath,
         optional: false,
         learningOutcomeId: null,
         learningOutcomeName: null,
         contentId: lens.id ?? null,
         tldr: lens.tldr,
-        videoId: videoId ?? null,
+        summaryForTutor: lens.summaryForTutor,
+        addToAiContext: lens.addToAiContext,
         ...computeSectionStats(segments),
+        displayType: computeDisplayType(segments),
     };
     const flattenedModule = {
-        slug: 'lens/' + fileNameToSlug(lensPath),
+        slug: "lens/" + fileNameToSlug(lensPath),
         title: meta.title ?? fileNameToSlug(lensPath),
         contentId: lens.id ?? null,
         sections: [section],
+        sourcePath: lensPath,
     };
     return { module: flattenedModule, modules: [flattenedModule], errors };
 }