lens-content-processor 0.2.2 → 0.4.0

package/dist/flattener/index.js

@@ -1,10 +1,35 @@
+import { checkTierViolation } from '../validator/tier.js';
 import { parseModule, parsePageSegments } from '../parser/module.js';
 import { parseLearningOutcome } from '../parser/learning-outcome.js';
 import { parseLens } from '../parser/lens.js';
 import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion } from '../parser/wikilink.js';
 import { parseFrontmatter } from '../parser/frontmatter.js';
-import { extractArticleExcerpt } from '../bundler/article.js';
+import { fileNameToSlug } from '../utils/slug.js';
+import { extractArticleExcerpt, bundleArticleWithCollapsed } from '../bundler/article.js';
 import { extractVideoExcerpt } from '../bundler/video.js';
+function countWords(text) {
+    return text.trim().split(/\s+/).filter(Boolean).length;
+}
+/** Compute word count and video duration from a section's segments. */
+function computeSectionStats(segments) {
+    let words = 0;
+    let videoSeconds = 0;
+    for (const seg of segments) {
+        if (seg.type === 'text' || seg.type === 'article-excerpt') {
+            words += countWords(seg.content);
+        }
+        else if (seg.type === 'video-excerpt') {
+            if (seg.to != null)
+                videoSeconds += seg.to - seg.from;
+        }
+    }
+    const stats = {};
+    if (words > 0)
+        stats.wordCount = words;
+    if (videoSeconds > 0)
+        stats.videoDurationSeconds = videoSeconds;
+    return stats;
+}
 /**
  * Extract YouTube video ID from a YouTube URL.
  *
@@ -20,6 +45,98 @@ function extractVideoIdFromUrl(url) {
     const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]+)/);
     return match ? match[1] : null;
 }
+export function isBoundary(item) {
+    return '__boundary' in item && item.__boundary === true;
+}
+function toKebabCase(str) {
+    return str.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
+}
+export function validateBoundaries(items, file) {
+    const errors = [];
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries)
+        return errors;
+    // Check: sections before first boundary
+    const firstBoundaryIdx = items.findIndex(isBoundary);
+    const sectionsBefore = items.slice(0, firstBoundaryIdx).filter(i => !isBoundary(i));
+    if (sectionsBefore.length > 0) {
+        errors.push({
+            file,
+            message: 'Content found outside submodule boundaries — all content must be inside a submodule',
+            severity: 'error',
+        });
+    }
+    // Check: consecutive boundaries and trailing boundary
+    for (let i = 0; i < items.length; i++) {
+        if (isBoundary(items[i])) {
+            // Find next non-boundary item
+            let nextNonBoundary = i + 1;
+            while (nextNonBoundary < items.length && isBoundary(items[nextNonBoundary])) {
+                nextNonBoundary++;
+            }
+            // If this boundary has no sections before next boundary or end
+            if (nextNonBoundary > i + 1 || nextNonBoundary >= items.length) {
+                // nextNonBoundary > i + 1 means consecutive boundaries
+                // nextNonBoundary >= items.length means trailing boundary with no content
+                if (nextNonBoundary > i + 1) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content before next submodule`,
+                        severity: 'error',
+                    });
+                }
+                else if (nextNonBoundary >= items.length) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content after this marker`,
+                        severity: 'error',
+                    });
+                }
+            }
+        }
+    }
+    return errors;
+}
+export function splitAtBoundaries(items, parentSlug, parentTitle) {
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries) {
+        // No split — return single group without parentSlug
+        return [{
+                slug: parentSlug,
+                title: parentTitle,
+                parentSlug: undefined,
+                parentTitle: undefined,
+                sections: items.filter(i => !isBoundary(i)),
+                contentId: null,
+            }];
+    }
+    const groups = [];
+    let currentGroup = null;
+    for (const item of items) {
+        if (isBoundary(item)) {
+            if (currentGroup) {
+                groups.push(currentGroup);
+            }
+            const slug = item.customSlug ?? toKebabCase(item.title);
+            currentGroup = {
+                slug: `${parentSlug}/${slug}`,
+                title: item.title,
+                parentSlug,
+                parentTitle,
+                sections: [],
+                contentId: null,
+            };
+        }
+        else if (currentGroup) {
+            currentGroup.sections.push(item);
+        }
+        // sections before first boundary are orphans — handled by validateBoundaries
+    }
+    if (currentGroup) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
 /**
  * Flatten a module by resolving all references to Learning Outcomes, Lenses, and content.
  *
@@ -34,11 +151,12 @@ function extractVideoIdFromUrl(url) {
  * @param visitedPaths - Optional set of already-visited paths for cycle detection
  * @returns Flattened module with resolved sections and segments, plus any errors
  */
-export function flattenModule(modulePath, files, visitedPaths = new Set()) {
+export function flattenModule(modulePath, files, visitedPaths = new Set(), tierMap) {
     // Check for circular reference
     if (visitedPaths.has(modulePath)) {
         return {
             module: null,
+            modules: [],
             errors: [{
                     file: modulePath,
                     message: `Circular reference detected: ${modulePath}`,
@@ -57,70 +175,295 @@ export function flattenModule(modulePath, files, visitedPaths = new Set()) {
             message: `Module file not found: ${modulePath}`,
             severity: 'error',
         });
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     // Parse the module
     const moduleResult = parseModule(moduleContent, modulePath);
     errors.push(...moduleResult.errors);
     if (!moduleResult.module) {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const parsedModule = moduleResult.module;
-    const flattenedSections = [];
-    // Process each section in the module
-    for (const section of parsedModule.sections) {
+    const flatItems = [];
+    // Helper: process a section (LO, Page, Uncategorized) and return Section[]
+    function processSection(section, subsectionLevel) {
         if (section.type === 'learning-outcome') {
-            // Resolve the Learning Outcome reference
-            // Create a copy of visitedPaths for this section's reference chain
-            // This allows the same file to be referenced in different sections
-            // while still detecting cycles within a single chain
             const sectionVisitedPaths = new Set(visitedPaths);
-            const result = flattenLearningOutcomeSection(section, modulePath, files, sectionVisitedPaths);
+            const result = flattenLearningOutcomeSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
+            return result.sections;
         }
         else if (section.type === 'page') {
-            // Page sections don't have LO references, they have inline content
-            // Parse the section body for ## Text subsections
-            const textResult = parsePageSegments(section.body, modulePath, section.line);
+            const level = subsectionLevel ?? 2;
+            const textResult = parsePageSegments(section.body, modulePath, section.line, level);
             errors.push(...textResult.errors);
-            const pageSection = {
-                type: 'page',
-                meta: { title: section.title },
-                segments: textResult.segments,
-                optional: section.fields.optional?.toLowerCase() === 'true',
-                contentId: section.fields.id ?? null,
-                learningOutcomeId: null,
-                learningOutcomeName: null,
-                videoId: null,
-            };
-            flattenedSections.push(pageSection);
+            return [{
+                    type: 'page',
+                    meta: { title: section.title },
+                    segments: textResult.segments,
+                    optional: section.fields.optional?.toLowerCase() === 'true',
+                    contentId: section.fields.id ?? null,
+                    learningOutcomeId: null,
+                    learningOutcomeName: null,
+                    videoId: null,
+                    ...computeSectionStats(textResult.segments),
+                }];
         }
         else if (section.type === 'uncategorized') {
-            // Uncategorized sections can contain ## Lens: references
-            // Each lens becomes its own section (lens-video, lens-article, or page)
             const sectionVisitedPaths = new Set(visitedPaths);
-            const result = flattenUncategorizedSection(section, modulePath, files, sectionVisitedPaths);
+            const result = flattenUncategorizedSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
+            return result.sections;
         }
+        return [];
     }
-    const flattenedModule = {
-        slug: parsedModule.slug,
-        title: parsedModule.title,
+    // Process each section in the module
+    for (const section of parsedModule.sections) {
+        if (section.type === 'submodule') {
+            // Emit boundary marker
+            flatItems.push({
+                __boundary: true,
+                title: section.title,
+                customSlug: section.fields.slug,
+            });
+            // Process children at level+1
+            if (section.children) {
+                for (const child of section.children) {
+                    const childSections = processSection(child, child.level + 1);
+                    flatItems.push(...childSections);
+                }
+            }
+        }
+        else {
+            // Check if this is an LO with submodules
+            if (section.type === 'learning-outcome') {
+                const loSections = processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors);
+                if (loSections) {
+                    flatItems.push(...loSections);
+                    continue;
+                }
+            }
+            const sections = processSection(section);
+            flatItems.push(...sections);
+        }
+    }
+    // Validate and split at boundaries
+    const boundaryErrors = validateBoundaries(flatItems, modulePath);
+    errors.push(...boundaryErrors);
+    const groups = splitAtBoundaries(flatItems, parsedModule.slug, parsedModule.title);
+    const resultModules = groups.map(g => ({
+        slug: g.slug,
+        title: g.title,
         contentId: parsedModule.contentId,
-        sections: flattenedSections,
+        sections: g.sections,
+        ...(g.parentSlug ? { parentSlug: g.parentSlug, parentTitle: g.parentTitle } : {}),
+        ...(moduleError ? { error: moduleError } : {}),
+    }));
+    // Backwards compat: module is the first result or a merged single module
+    const primaryModule = resultModules.length > 0 ? resultModules[0] : null;
+    return { module: resultModules.length === 1 && !resultModules[0].parentSlug ? resultModules[0] : primaryModule, modules: resultModules, errors };
+}
+/**
+ * Build a test Section from a ParsedTestRef with inline segments.
+ * Shared by both processLOWithSubmodules() and flattenLearningOutcomeSection().
+ */
+function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, tierMap, errors) {
+    if (!testRef.segments.length)
+        return null;
+    const testSegments = [];
+    const stubLensSection = {
+        type: 'page',
+        title: 'Test',
+        segments: [],
+        line: 0,
     };
-    if (moduleError) {
-        flattenedModule.error = moduleError;
+    for (const parsedSegment of testRef.segments) {
+        const segmentResult = convertSegment(parsedSegment, stubLensSection, loPath, files, new Set(visitedPaths), tierMap);
+        errors.push(...segmentResult.errors);
+        if (segmentResult.segment) {
+            testSegments.push(segmentResult.segment);
+        }
+    }
+    if (testSegments.length === 0)
+        return null;
+    const hasFeedback = testSegments.some((s) => s.type === 'question' && s.feedback);
+    return {
+        type: 'test',
+        meta: { title: 'Test' },
+        segments: testSegments,
+        optional: false,
+        ...(hasFeedback && { feedback: true }),
+        contentId: null,
+        learningOutcomeId: loId,
+        learningOutcomeName: loName,
+        videoId: null,
+    };
+}
+/**
+ * Check if a Learning Outcome section resolves to an LO with submodules.
+ * If so, emit boundary markers + sections. Returns null if no submodules.
+ */
+function processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors) {
+    // Resolve the LO file to check for submodules
+    const source = section.fields.source;
+    if (!source)
+        return null;
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error)
+        return null;
+    const loPathResolved = resolveWikilinkPath(wikilink.path, modulePath);
+    const loPath = findFileWithExtension(loPathResolved, files);
+    if (!loPath)
+        return null;
+    const loContent = files.get(loPath);
+    if (!loContent)
+        return null;
+    const loResult = parseLearningOutcome(loContent, loPath);
+    if (!loResult.learningOutcome?.submodules)
+        return null;
+    // This LO has submodules — emit boundaries and resolve lenses
+    const items = [];
+    for (const sub of loResult.learningOutcome.submodules) {
+        items.push({
+            __boundary: true,
+            title: sub.title,
+            customSlug: sub.customSlug,
+        });
+        // Resolve each lens in this submodule group
+        for (const lensRef of sub.lenses) {
+            const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
+            if (!lensPath) {
+                const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
+                const suggestion = formatSuggestion(similarFiles, loPath) ?? 'Check the file path in the wiki-link';
+                errors.push({
+                    file: loPath,
+                    message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
+                    suggestion,
+                    severity: 'error',
+                });
+                continue;
+            }
+            // Check tier violation
+            if (tierMap) {
+                const parentTier = tierMap.get(loPath) ?? 'production';
+                const childTier = tierMap.get(lensPath) ?? 'production';
+                const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+                if (violation) {
+                    errors.push(violation);
+                }
+                if (childTier === 'ignored')
+                    continue;
+            }
+            if (visitedPaths.has(lensPath)) {
+                errors.push({
+                    file: loPath,
+                    message: `Circular reference detected: ${lensPath}`,
+                    severity: 'error',
+                });
+                continue;
+            }
+            const sectionVisitedPaths = new Set(visitedPaths);
+            sectionVisitedPaths.add(lensPath);
+            const lensContent = files.get(lensPath);
+            const lensResult = parseLens(lensContent, lensPath);
+            errors.push(...lensResult.errors);
+            if (!lensResult.lens)
+                continue;
+            const lens = lensResult.lens;
+            let sectionType = 'page';
+            const meta = { title: section.title };
+            const segments = [];
+            let videoId;
+            for (const lensSection of lens.sections) {
+                if (lensSection.type === 'lens-article') {
+                    sectionType = 'lens-article';
+                    if (lensSection.source) {
+                        const articleWikilink = parseWikilink(lensSection.source);
+                        if (articleWikilink && !articleWikilink.error) {
+                            const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
+                            const articlePath = findFileWithExtension(articlePathResolved, files);
+                            if (articlePath) {
+                                const articleContent = files.get(articlePath);
+                                const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
+                                if (articleFrontmatter.frontmatter.title)
+                                    meta.title = articleFrontmatter.frontmatter.title;
+                                if (articleFrontmatter.frontmatter.author) {
+                                    const raw = articleFrontmatter.frontmatter.author;
+                                    meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
+                                }
+                                if (articleFrontmatter.frontmatter.source_url)
+                                    meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
+                            }
+                        }
+                    }
+                }
+                else if (lensSection.type === 'page') {
+                    sectionType = 'page';
+                    if (lensSection.title)
+                        meta.title = lensSection.title;
+                }
+                else if (lensSection.type === 'lens-video') {
+                    sectionType = 'lens-video';
+                    if (lensSection.source) {
+                        const videoWikilink = parseWikilink(lensSection.source);
+                        if (videoWikilink && !videoWikilink.error) {
+                            const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
+                            const videoPath = findFileWithExtension(videoPathResolved, files);
+                            if (videoPath) {
+                                const videoContent = files.get(videoPath);
+                                const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
+                                if (videoFrontmatter.frontmatter.title)
+                                    meta.title = videoFrontmatter.frontmatter.title;
+                                if (videoFrontmatter.frontmatter.channel)
+                                    meta.channel = videoFrontmatter.frontmatter.channel;
+                                if (videoFrontmatter.frontmatter.url) {
+                                    const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
+                                    if (extractedVideoId)
+                                        videoId = extractedVideoId;
+                                }
+                            }
+                        }
+                    }
+                }
+                for (const parsedSegment of lensSection.segments) {
+                    const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, sectionVisitedPaths, tierMap);
+                    errors.push(...segmentResult.errors);
+                    if (segmentResult.segment)
+                        segments.push(segmentResult.segment);
+                }
+                applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
+            }
+            const lo = loResult.learningOutcome;
+            items.push({
+                type: sectionType,
+                meta,
+                segments,
+                optional: section.fields.optional?.toLowerCase() === 'true' || lensRef.optional,
+                learningOutcomeId: lo.id ?? null,
+                learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
+                contentId: lens.id ?? null,
+                tldr: lens.tldr,
+                videoId: videoId ?? null,
+                ...computeSectionStats(segments),
+            });
+        }
+        // Process test section if present in this submodule
+        if (sub.test) {
+            const lo = loResult.learningOutcome;
+            const testSection = buildTestSection(sub.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+            if (testSection)
+                items.push(testSection);
+        }
     }
-    return { module: flattenedModule, errors };
+    // Also emit LO parse errors (we skipped earlier since we returned non-null)
+    errors.push(...loResult.errors);
+    return items;
 }
 /**
  * Flatten a Learning Outcome section by resolving its LO file and all referenced lenses.
  * Each lens in the LO becomes its own section.
  */
-function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths) {
+function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths, tierMap) {
     const errors = [];
     const sections = [];
     // Get the source wikilink
@@ -144,7 +487,9 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
         errors.push({
             file: modulePath,
             line: section.line,
-            message: `Invalid wikilink format: ${source}`,
+            message: wikilink?.error
+                ? `${wikilink.error}: ${source}`
+                : `Invalid wikilink format: ${source}`,
             suggestion,
             severity: 'error',
         });
@@ -166,6 +511,19 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
         });
         return { sections: [], errors };
     }
+    // Check tier violation (module → LO)
+    if (tierMap) {
+        const parentTier = tierMap.get(modulePath) ?? 'production';
+        const childTier = tierMap.get(loPath) ?? 'production';
+        const violation = checkTierViolation(modulePath, parentTier, loPath, childTier, 'learning outcome', section.line);
+        if (violation) {
+            errors.push(violation);
+        }
+        // Skip ignored children silently (they're not processed)
+        if (childTier === 'ignored') {
+            return { sections: [], errors };
+        }
+    }
     // Check for circular reference
     if (visitedPaths.has(loPath)) {
         errors.push({
@@ -200,6 +558,18 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
             });
             continue;
         }
+        // Check tier violation (LO → Lens)
+        if (tierMap) {
+            const parentTier = tierMap.get(loPath) ?? 'production';
+            const childTier = tierMap.get(lensPath) ?? 'production';
+            const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+            if (violation) {
+                errors.push(violation);
+            }
+            if (childTier === 'ignored') {
+                continue;
+            }
+        }
         // Check for circular reference
         if (visitedPaths.has(lensPath)) {
             errors.push({
@@ -242,11 +612,15 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
                                 meta.title = articleFrontmatter.frontmatter.title;
                             }
                             if (articleFrontmatter.frontmatter.author) {
-                                meta.author = articleFrontmatter.frontmatter.author;
+                                const raw = articleFrontmatter.frontmatter.author;
+                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
                             }
                             if (articleFrontmatter.frontmatter.source_url) {
                                 meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
                             }
+                            if (articleFrontmatter.frontmatter.published) {
+                                meta.published = String(articleFrontmatter.frontmatter.published);
+                            }
                         }
                     }
                 }
@@ -289,12 +663,14 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
             }
             // Process segments
             for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths);
+                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
                 errors.push(...segmentResult.errors);
                 if (segmentResult.segment) {
                     segments.push(segmentResult.segment);
                 }
             }
+            // Apply collapsed content to article-excerpt segments
+            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
         }
         // Create a section for this lens
         // Optional can come from either:
@@ -308,17 +684,25 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths)
             learningOutcomeId: lo.id ?? null,
             learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
             contentId: lens.id ?? null,
+            tldr: lens.tldr,
             videoId: videoId ?? null,
+            ...computeSectionStats(segments),
         };
         sections.push(resultSection);
     }
+    // After lens processing, add test section if present with inline segments
+    if (lo.test) {
+        const testSection = buildTestSection(lo.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+        if (testSection)
+            sections.push(testSection);
+    }
     return { sections, errors };
 }
 /**
  * Flatten an Uncategorized section by parsing its ## Lens: references.
  * Each lens becomes its own section with the appropriate type (lens-video, lens-article, or page).
  */
-function flattenUncategorizedSection(section, modulePath, files, visitedPaths) {
+function flattenUncategorizedSection(section, modulePath, files, visitedPaths, tierMap) {
     const errors = [];
     const sections = [];
     // Parse the section body for ## Lens: subsections
@@ -349,6 +733,18 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths) {
             });
             continue;
         }
+        // Check tier violation (Uncategorized/Module → Lens)
+        if (tierMap) {
+            const parentTier = tierMap.get(modulePath) ?? 'production';
+            const childTier = tierMap.get(lensPath) ?? 'production';
+            const violation = checkTierViolation(modulePath, parentTier, lensPath, childTier, 'lens');
+            if (violation) {
+                errors.push(violation);
+            }
+            if (childTier === 'ignored') {
+                continue;
+            }
+        }
         // Check for circular reference
         if (visitedPaths.has(lensPath)) {
             errors.push({
@@ -391,11 +787,15 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths) {
                                 meta.title = articleFrontmatter.frontmatter.title;
                             }
                             if (articleFrontmatter.frontmatter.author) {
-                                meta.author = articleFrontmatter.frontmatter.author;
+                                const raw = articleFrontmatter.frontmatter.author;
+                                meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
                             }
                             if (articleFrontmatter.frontmatter.source_url) {
                                 meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
                             }
+                            if (articleFrontmatter.frontmatter.published) {
+                                meta.published = String(articleFrontmatter.frontmatter.published);
+                            }
                         }
                     }
                 }
@@ -438,12 +838,14 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths) {
             }
             // Process segments
             for (const parsedSegment of lensSection.segments) {
-                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths);
+                const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
                 errors.push(...segmentResult.errors);
                 if (segmentResult.segment) {
                     segments.push(segmentResult.segment);
                 }
             }
+            // Apply collapsed content to article-excerpt segments
+            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
         }
         // Create a section for this lens
         const resultSection = {
@@ -454,7 +856,9 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths) {
             learningOutcomeId: null,
             learningOutcomeName: null,
             contentId: lens.id ?? null,
+            tldr: lens.tldr,
             videoId: videoId ?? null,
+            ...computeSectionStats(segments),
         };
         sections.push(resultSection);
     }
@@ -570,11 +974,64 @@ function parseUncategorizedLensRefs(body, parentPath) {
     }
     return lensRefs;
 }
+/**
+ * After converting segments, apply collapsed_before/collapsed_after to article-excerpt segments.
+ * This resolves the article source once and calls bundleArticleWithCollapsed to compute
+ * which parts of the article are outside the excerpted ranges.
+ */
+function applyCollapsedContent(segments, parsedSegments, lensSection, lensPath, files) {
+    // Only process lens-article sections with article-excerpt segments
+    if (lensSection.type !== 'lens-article' || !lensSection.source)
+        return;
+    // Collect article-excerpt segment indices
+    const excerptIndices = [];
+    for (let i = 0; i < segments.length; i++) {
+        if (segments[i].type === 'article-excerpt') {
+            excerptIndices.push(i);
+        }
+    }
+    if (excerptIndices.length === 0)
+        return;
+    // Map to parsed segments to get anchors (order-preserving)
+    const parsedExcerpts = parsedSegments.filter(ps => ps.type === 'article-excerpt');
+    const excerptInfos = excerptIndices.map((segIdx, i) => ({
+        index: segIdx,
+        from: parsedExcerpts[i]?.fromAnchor,
+        to: parsedExcerpts[i]?.toAnchor,
+    }));
+    // Only apply collapsed if at least one excerpt has anchors
+    const hasAnchors = excerptInfos.some(e => e.from || e.to);
+    if (!hasAnchors)
+        return;
+    // Resolve article path
+    const wikilink = parseWikilink(lensSection.source);
+    if (!wikilink || wikilink.error)
+        return;
+    const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
+    const articlePath = findFileWithExtension(articlePathResolved, files);
+    if (!articlePath)
+        return;
+    const articleContent = files.get(articlePath);
+    // Call bundleArticleWithCollapsed
+    const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map(e => ({ from: e.from, to: e.to })), articlePath);
+    // Overlay collapsed_before/collapsed_after onto segments
+    for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
+        const segIdx = excerptInfos[i].index;
+        const collapsed = collapsedResults[i];
+        const segment = segments[segIdx];
+        if (collapsed.collapsed_before) {
+            segment.collapsed_before = collapsed.collapsed_before;
+        }
+        if (collapsed.collapsed_after) {
+            segment.collapsed_after = collapsed.collapsed_after;
+        }
+    }
+}
 /**
  * Convert a parsed lens segment into a final flattened segment.
  * For article-excerpt and video-excerpt, this involves extracting content from source files.
  */
-function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths) {
+function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap) {
     const errors = [];
     switch (parsedSegment.type) {
         case 'text': {
@@ -642,6 +1099,18 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 });
                 return { segment: null, errors };
             }
+            // Check tier violation (Lens → Article)
+            if (tierMap) {
+                const parentTier = tierMap.get(lensPath) ?? 'production';
+                const childTier = tierMap.get(articlePath) ?? 'production';
+                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, 'article');
+                if (violation) {
+                    errors.push(violation);
+                }
+                if (childTier === 'ignored') {
+                    return { segment: null, errors };
+                }
+            }
             // Check if the article path points back to an already-visited structural file
             // This would indicate a circular reference (e.g., a lens source pointing back to an LO)
             // Note: We only check, we don't add article paths to visitedPaths since
@@ -658,7 +1127,7 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             // Extract the excerpt
             const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
@@ -707,6 +1176,18 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 });
                 return { segment: null, errors };
             }
+            // Check tier violation (Lens → Video)
+            if (tierMap) {
+                const parentTier = tierMap.get(lensPath) ?? 'production';
+                const childTier = tierMap.get(videoPath) ?? 'production';
+                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, 'video transcript');
+                if (violation) {
+                    errors.push(violation);
+                }
+                if (childTier === 'ignored') {
+                    return { segment: null, errors };
+                }
+            }
             // Check if the video path points back to an already-visited structural file
             // This would indicate a circular reference
             // Note: We only check, we don't add video paths to visitedPaths since
@@ -740,7 +1221,7 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             // Extract the video excerpt
             const excerptResult = extractVideoExcerpt(transcriptContent, parsedSegment.fromTimeStr, parsedSegment.toTimeStr, videoPath, timestamps);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
@@ -754,8 +1235,178 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             }
             return { segment, errors };
         }
+        case 'question': {
+            const segment = {
+                type: 'question',
+                content: parsedSegment.content,
+            };
+            if (parsedSegment.assessmentInstructions)
+                segment.assessmentInstructions = parsedSegment.assessmentInstructions;
+            if (parsedSegment.maxTime)
+                segment.maxTime = parsedSegment.maxTime;
+            if (parsedSegment.maxChars !== undefined)
+                segment.maxChars = parsedSegment.maxChars;
+            if (parsedSegment.enforceVoice)
+                segment.enforceVoice = true;
+            if (parsedSegment.optional)
+                segment.optional = true;
+            if (parsedSegment.feedback)
+                segment.feedback = true;
+            return { segment, errors };
+        }
+        case 'roleplay': {
+            const segment = {
+                type: 'roleplay',
+                id: parsedSegment.id,
+                content: parsedSegment.content,
+                aiInstructions: parsedSegment.aiInstructions,
+            };
+            if (parsedSegment.openingMessage)
+                segment.openingMessage = parsedSegment.openingMessage;
+            if (parsedSegment.assessmentInstructions)
+                segment.assessmentInstructions = parsedSegment.assessmentInstructions;
+            if (parsedSegment.optional)
+                segment.optional = true;
+            if (parsedSegment.feedback)
+                segment.feedback = true;
+            return { segment, errors };
+        }
         default:
             return { segment: null, errors };
     }
 }
+/**
+ * Flatten a single Lens file into a FlattenedModule.
+ *
+ * This wraps a standalone Lens as a single-section module so it can be
+ * rendered by the frontend using the existing Module.tsx component.
+ *
+ * The resulting module has:
+ * - slug: 'lens/' + fileNameToSlug(lensPath)
+ * - title: extracted from source metadata or lens header
+ * - sections: exactly one section (lens-article, lens-video, or page)
+ *
+ * @param lensPath - Path to the lens file within the files Map
+ * @param files - Map of all file paths to their content
+ * @param tierMap - Optional tier map for filtering ignored content
+ * @param preParsedLens - Optional pre-parsed lens to avoid re-parsing
+ * @returns Flattened module with a single section, plus any errors
+ */
+export function flattenLens(lensPath, files, tierMap, preParsedLens) {
+    const errors = [];
+    // Skip ignored lenses
+    if (tierMap?.get(lensPath) === 'ignored') {
+        return { module: null, modules: [], errors };
+    }
+    const lensContent = files.get(lensPath);
+    if (!lensContent) {
+        errors.push({
+            file: lensPath,
+            message: `Lens file not found: ${lensPath}`,
+            severity: 'error',
+        });
+        return { module: null, modules: [], errors };
+    }
+    // Use pre-parsed lens if provided, otherwise parse
+    const lens = preParsedLens ?? (() => {
+        const lensResult = parseLens(lensContent, lensPath);
+        errors.push(...lensResult.errors);
+        return lensResult.lens;
+    })();
+    if (!lens) {
+        return { module: null, modules: [], errors };
+    }
+    const visitedPaths = new Set([lensPath]);
+    // Process lens sections into a single flattened Section
+    let sectionType = 'page';
+    const meta = {};
+    const segments = [];
+    let videoId;
+    for (const lensSection of lens.sections) {
+        if (lensSection.type === 'lens-article') {
+            sectionType = 'lens-article';
+            // Extract article metadata from the article file's frontmatter
+            if (lensSection.source) {
+                const articleWikilink = parseWikilink(lensSection.source);
+                if (articleWikilink && !articleWikilink.error) {
+                    const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
+                    const articlePath = findFileWithExtension(articlePathResolved, files);
+                    if (articlePath) {
+                        const articleContent = files.get(articlePath);
+                        const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
+                        if (articleFrontmatter.frontmatter.title)
+                            meta.title = articleFrontmatter.frontmatter.title;
+                        if (articleFrontmatter.frontmatter.author)
+                            meta.author = articleFrontmatter.frontmatter.author;
+                        if (articleFrontmatter.frontmatter.source_url)
+                            meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
+                        if (articleFrontmatter.frontmatter.published)
+                            meta.published = String(articleFrontmatter.frontmatter.published);
+                    }
+                }
+            }
+        }
+        else if (lensSection.type === 'page') {
+            sectionType = 'page';
+            if (lensSection.title)
+                meta.title = lensSection.title;
+        }
+        else if (lensSection.type === 'lens-video') {
+            sectionType = 'lens-video';
+            // Extract video metadata from the video transcript file's frontmatter
+            if (lensSection.source) {
+                const videoWikilink = parseWikilink(lensSection.source);
+                if (videoWikilink && !videoWikilink.error) {
+                    const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
+                    const videoPath = findFileWithExtension(videoPathResolved, files);
+                    if (videoPath) {
+                        const videoContent = files.get(videoPath);
+                        const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
+                        if (videoFrontmatter.frontmatter.title)
+                            meta.title = videoFrontmatter.frontmatter.title;
+                        if (videoFrontmatter.frontmatter.channel)
+                            meta.channel = videoFrontmatter.frontmatter.channel;
+                        if (videoFrontmatter.frontmatter.url) {
+                            const extractedId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
+                            if (extractedId)
+                                videoId = extractedId;
+                        }
+                    }
+                }
+            }
+        }
+        // Process segments
+        for (const parsedSegment of lensSection.segments) {
+            const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, visitedPaths, tierMap);
+            errors.push(...segmentResult.errors);
+            if (segmentResult.segment)
+                segments.push(segmentResult.segment);
+        }
+        // Apply collapsed content to article-excerpt segments
+        applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
+    }
+    // Fallback title from filename if not extracted from source metadata
+    if (!meta.title) {
+        meta.title = fileNameToSlug(lensPath).replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
+    }
+    const section = {
+        type: sectionType,
+        meta,
+        segments,
+        optional: false,
+        learningOutcomeId: null,
+        learningOutcomeName: null,
+        contentId: lens.id ?? null,
+        tldr: lens.tldr,
+        videoId: videoId ?? null,
+        ...computeSectionStats(segments),
+    };
+    const flattenedModule = {
+        slug: 'lens/' + fileNameToSlug(lensPath),
+        title: meta.title ?? fileNameToSlug(lensPath),
+        contentId: lens.id ?? null,
+        sections: [section],
+    };
+    return { module: flattenedModule, modules: [flattenedModule], errors };
+}
 //# sourceMappingURL=index.js.map