lens-content-processor 0.3.0 → 0.4.0

package/dist/flattener/index.js

@@ -5,8 +5,31 @@ import { parseLens } from '../parser/lens.js';
 import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion } from '../parser/wikilink.js';
 import { parseFrontmatter } from '../parser/frontmatter.js';
 import { fileNameToSlug } from '../utils/slug.js';
-import { extractArticleExcerpt } from '../bundler/article.js';
+import { extractArticleExcerpt, bundleArticleWithCollapsed } from '../bundler/article.js';
 import { extractVideoExcerpt } from '../bundler/video.js';
+function countWords(text) {
+    return text.trim().split(/\s+/).filter(Boolean).length;
+}
+/** Compute word count and video duration from a section's segments. */
+function computeSectionStats(segments) {
+    let words = 0;
+    let videoSeconds = 0;
+    for (const seg of segments) {
+        if (seg.type === 'text' || seg.type === 'article-excerpt') {
+            words += countWords(seg.content);
+        }
+        else if (seg.type === 'video-excerpt') {
+            if (seg.to != null)
+                videoSeconds += seg.to - seg.from;
+        }
+    }
+    const stats = {};
+    if (words > 0)
+        stats.wordCount = words;
+    if (videoSeconds > 0)
+        stats.videoDurationSeconds = videoSeconds;
+    return stats;
+}
 /**
  * Extract YouTube video ID from a YouTube URL.
  *
@@ -22,6 +45,98 @@ function extractVideoIdFromUrl(url) {
     const match = url.match(/(?:youtube\.com\/watch\?v=|youtu\.be\/|youtube\.com\/embed\/)([a-zA-Z0-9_-]+)/);
     return match ? match[1] : null;
 }
+export function isBoundary(item) {
+    return '__boundary' in item && item.__boundary === true;
+}
+function toKebabCase(str) {
+    return str.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
+}
+export function validateBoundaries(items, file) {
+    const errors = [];
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries)
+        return errors;
+    // Check: sections before first boundary
+    const firstBoundaryIdx = items.findIndex(isBoundary);
+    const sectionsBefore = items.slice(0, firstBoundaryIdx).filter(i => !isBoundary(i));
+    if (sectionsBefore.length > 0) {
+        errors.push({
+            file,
+            message: 'Content found outside submodule boundaries — all content must be inside a submodule',
+            severity: 'error',
+        });
+    }
+    // Check: consecutive boundaries and trailing boundary
+    for (let i = 0; i < items.length; i++) {
+        if (isBoundary(items[i])) {
+            // Find next non-boundary item
+            let nextNonBoundary = i + 1;
+            while (nextNonBoundary < items.length && isBoundary(items[nextNonBoundary])) {
+                nextNonBoundary++;
+            }
+            // If this boundary has no sections before next boundary or end
+            if (nextNonBoundary > i + 1 || nextNonBoundary >= items.length) {
+                // nextNonBoundary > i + 1 means consecutive boundaries
+                // nextNonBoundary >= items.length means trailing boundary with no content
+                if (nextNonBoundary > i + 1) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content before next submodule`,
+                        severity: 'error',
+                    });
+                }
+                else if (nextNonBoundary >= items.length) {
+                    errors.push({
+                        file,
+                        message: `Submodule "${items[i].title}" is empty — no content after this marker`,
+                        severity: 'error',
+                    });
+                }
+            }
+        }
+    }
+    return errors;
+}
+export function splitAtBoundaries(items, parentSlug, parentTitle) {
+    const hasBoundaries = items.some(isBoundary);
+    if (!hasBoundaries) {
+        // No split — return single group without parentSlug
+        return [{
+                slug: parentSlug,
+                title: parentTitle,
+                parentSlug: undefined,
+                parentTitle: undefined,
+                sections: items.filter(i => !isBoundary(i)),
+                contentId: null,
+            }];
+    }
+    const groups = [];
+    let currentGroup = null;
+    for (const item of items) {
+        if (isBoundary(item)) {
+            if (currentGroup) {
+                groups.push(currentGroup);
+            }
+            const slug = item.customSlug ?? toKebabCase(item.title);
+            currentGroup = {
+                slug: `${parentSlug}/${slug}`,
+                title: item.title,
+                parentSlug,
+                parentTitle,
+                sections: [],
+                contentId: null,
+            };
+        }
+        else if (currentGroup) {
+            currentGroup.sections.push(item);
+        }
+        // sections before first boundary are orphans — handled by validateBoundaries
+    }
+    if (currentGroup) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
 /**
  * Flatten a module by resolving all references to Learning Outcomes, Lenses, and content.
  *
@@ -41,6 +156,7 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
     if (visitedPaths.has(modulePath)) {
         return {
             module: null,
+            modules: [],
             errors: [{
                     file: modulePath,
                     message: `Circular reference detected: ${modulePath}`,
@@ -59,64 +175,289 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
             message: `Module file not found: ${modulePath}`,
             severity: 'error',
         });
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     // Parse the module
     const moduleResult = parseModule(moduleContent, modulePath);
     errors.push(...moduleResult.errors);
     if (!moduleResult.module) {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const parsedModule = moduleResult.module;
-    const flattenedSections = [];
-    // Process each section in the module
-    for (const section of parsedModule.sections) {
+    const flatItems = [];
+    // Helper: process a section (LO, Page, Uncategorized) and return Section[]
+    function processSection(section, subsectionLevel) {
         if (section.type === 'learning-outcome') {
-            // Resolve the Learning Outcome reference
-            // Create a copy of visitedPaths for this section's reference chain
-            // This allows the same file to be referenced in different sections
-            // while still detecting cycles within a single chain
             const sectionVisitedPaths = new Set(visitedPaths);
             const result = flattenLearningOutcomeSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
+            return result.sections;
         }
         else if (section.type === 'page') {
-            // Page sections don't have LO references, they have inline content
-            // Parse the section body for ## Text subsections
-            const textResult = parsePageSegments(section.body, modulePath, section.line);
+            const level = subsectionLevel ?? 2;
+            const textResult = parsePageSegments(section.body, modulePath, section.line, level);
             errors.push(...textResult.errors);
-            const pageSection = {
-                type: 'page',
-                meta: { title: section.title },
-                segments: textResult.segments,
-                optional: section.fields.optional?.toLowerCase() === 'true',
-                contentId: section.fields.id ?? null,
-                learningOutcomeId: null,
-                learningOutcomeName: null,
-                videoId: null,
-            };
-            flattenedSections.push(pageSection);
+            return [{
+                    type: 'page',
+                    meta: { title: section.title },
+                    segments: textResult.segments,
+                    optional: section.fields.optional?.toLowerCase() === 'true',
+                    contentId: section.fields.id ?? null,
+                    learningOutcomeId: null,
+                    learningOutcomeName: null,
+                    videoId: null,
+                    ...computeSectionStats(textResult.segments),
+                }];
         }
         else if (section.type === 'uncategorized') {
-            // Uncategorized sections can contain ## Lens: references
-            // Each lens becomes its own section (lens-video, lens-article, or page)
             const sectionVisitedPaths = new Set(visitedPaths);
             const result = flattenUncategorizedSection(section, modulePath, files, sectionVisitedPaths, tierMap);
             errors.push(...result.errors);
-            flattenedSections.push(...result.sections);
+            return result.sections;
         }
+        return [];
     }
-    const flattenedModule = {
-        slug: parsedModule.slug,
-        title: parsedModule.title,
+    // Process each section in the module
+    for (const section of parsedModule.sections) {
+        if (section.type === 'submodule') {
+            // Emit boundary marker
+            flatItems.push({
+                __boundary: true,
+                title: section.title,
+                customSlug: section.fields.slug,
+            });
+            // Process children at level+1
+            if (section.children) {
+                for (const child of section.children) {
+                    const childSections = processSection(child, child.level + 1);
+                    flatItems.push(...childSections);
+                }
+            }
+        }
+        else {
+            // Check if this is an LO with submodules
+            if (section.type === 'learning-outcome') {
+                const loSections = processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors);
+                if (loSections) {
+                    flatItems.push(...loSections);
+                    continue;
+                }
+            }
+            const sections = processSection(section);
+            flatItems.push(...sections);
+        }
+    }
+    // Validate and split at boundaries
+    const boundaryErrors = validateBoundaries(flatItems, modulePath);
+    errors.push(...boundaryErrors);
+    const groups = splitAtBoundaries(flatItems, parsedModule.slug, parsedModule.title);
+    const resultModules = groups.map(g => ({
+        slug: g.slug,
+        title: g.title,
         contentId: parsedModule.contentId,
-        sections: flattenedSections,
+        sections: g.sections,
+        ...(g.parentSlug ? { parentSlug: g.parentSlug, parentTitle: g.parentTitle } : {}),
+        ...(moduleError ? { error: moduleError } : {}),
+    }));
+    // Backwards compat: module is the first result or a merged single module
+    const primaryModule = resultModules.length > 0 ? resultModules[0] : null;
+    return { module: resultModules.length === 1 && !resultModules[0].parentSlug ? resultModules[0] : primaryModule, modules: resultModules, errors };
+}
+/**
+ * Build a test Section from a ParsedTestRef with inline segments.
+ * Shared by both processLOWithSubmodules() and flattenLearningOutcomeSection().
+ */
+function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, tierMap, errors) {
+    if (!testRef.segments.length)
+        return null;
+    const testSegments = [];
+    const stubLensSection = {
+        type: 'page',
+        title: 'Test',
+        segments: [],
+        line: 0,
     };
-    if (moduleError) {
-        flattenedModule.error = moduleError;
+    for (const parsedSegment of testRef.segments) {
+        const segmentResult = convertSegment(parsedSegment, stubLensSection, loPath, files, new Set(visitedPaths), tierMap);
+        errors.push(...segmentResult.errors);
+        if (segmentResult.segment) {
+            testSegments.push(segmentResult.segment);
+        }
     }
-    return { module: flattenedModule, errors };
+    if (testSegments.length === 0)
+        return null;
+    const hasFeedback = testSegments.some((s) => s.type === 'question' && s.feedback);
+    return {
+        type: 'test',
+        meta: { title: 'Test' },
+        segments: testSegments,
+        optional: false,
+        ...(hasFeedback && { feedback: true }),
+        contentId: null,
+        learningOutcomeId: loId,
+        learningOutcomeName: loName,
+        videoId: null,
+    };
+}
+/**
+ * Check if a Learning Outcome section resolves to an LO with submodules.
+ * If so, emit boundary markers + sections. Returns null if no submodules.
+ */
+function processLOWithSubmodules(section, modulePath, files, visitedPaths, tierMap, errors) {
+    // Resolve the LO file to check for submodules
+    const source = section.fields.source;
+    if (!source)
+        return null;
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error)
+        return null;
+    const loPathResolved = resolveWikilinkPath(wikilink.path, modulePath);
+    const loPath = findFileWithExtension(loPathResolved, files);
+    if (!loPath)
+        return null;
+    const loContent = files.get(loPath);
+    if (!loContent)
+        return null;
+    const loResult = parseLearningOutcome(loContent, loPath);
+    if (!loResult.learningOutcome?.submodules)
+        return null;
+    // This LO has submodules — emit boundaries and resolve lenses
+    const items = [];
+    for (const sub of loResult.learningOutcome.submodules) {
+        items.push({
+            __boundary: true,
+            title: sub.title,
+            customSlug: sub.customSlug,
+        });
+        // Resolve each lens in this submodule group
+        for (const lensRef of sub.lenses) {
+            const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
+            if (!lensPath) {
+                const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, 'Lenses');
+                const suggestion = formatSuggestion(similarFiles, loPath) ?? 'Check the file path in the wiki-link';
+                errors.push({
+                    file: loPath,
+                    message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
+                    suggestion,
+                    severity: 'error',
+                });
+                continue;
+            }
+            // Check tier violation
+            if (tierMap) {
+                const parentTier = tierMap.get(loPath) ?? 'production';
+                const childTier = tierMap.get(lensPath) ?? 'production';
+                const violation = checkTierViolation(loPath, parentTier, lensPath, childTier, 'lens');
+                if (violation) {
+                    errors.push(violation);
+                }
+                if (childTier === 'ignored')
+                    continue;
+            }
+            if (visitedPaths.has(lensPath)) {
+                errors.push({
+                    file: loPath,
+                    message: `Circular reference detected: ${lensPath}`,
+                    severity: 'error',
+                });
+                continue;
+            }
+            const sectionVisitedPaths = new Set(visitedPaths);
+            sectionVisitedPaths.add(lensPath);
+            const lensContent = files.get(lensPath);
+            const lensResult = parseLens(lensContent, lensPath);
+            errors.push(...lensResult.errors);
+            if (!lensResult.lens)
+                continue;
+            const lens = lensResult.lens;
+            let sectionType = 'page';
+            const meta = { title: section.title };
+            const segments = [];
+            let videoId;
+            for (const lensSection of lens.sections) {
+                if (lensSection.type === 'lens-article') {
+                    sectionType = 'lens-article';
+                    if (lensSection.source) {
+                        const articleWikilink = parseWikilink(lensSection.source);
+                        if (articleWikilink && !articleWikilink.error) {
+                            const articlePathResolved = resolveWikilinkPath(articleWikilink.path, lensPath);
+                            const articlePath = findFileWithExtension(articlePathResolved, files);
+                            if (articlePath) {
+                                const articleContent = files.get(articlePath);
+                                const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
+                                if (articleFrontmatter.frontmatter.title)
+                                    meta.title = articleFrontmatter.frontmatter.title;
+                                if (articleFrontmatter.frontmatter.author) {
+                                    const raw = articleFrontmatter.frontmatter.author;
+                                    meta.author = Array.isArray(raw) ? raw.join(', ') : String(raw);
+                                }
+                                if (articleFrontmatter.frontmatter.source_url)
+                                    meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
+                            }
+                        }
+                    }
+                }
+                else if (lensSection.type === 'page') {
+                    sectionType = 'page';
+                    if (lensSection.title)
+                        meta.title = lensSection.title;
+                }
+                else if (lensSection.type === 'lens-video') {
+                    sectionType = 'lens-video';
+                    if (lensSection.source) {
+                        const videoWikilink = parseWikilink(lensSection.source);
+                        if (videoWikilink && !videoWikilink.error) {
+                            const videoPathResolved = resolveWikilinkPath(videoWikilink.path, lensPath);
+                            const videoPath = findFileWithExtension(videoPathResolved, files);
+                            if (videoPath) {
+                                const videoContent = files.get(videoPath);
+                                const videoFrontmatter = parseFrontmatter(videoContent, videoPath);
+                                if (videoFrontmatter.frontmatter.title)
+                                    meta.title = videoFrontmatter.frontmatter.title;
+                                if (videoFrontmatter.frontmatter.channel)
+                                    meta.channel = videoFrontmatter.frontmatter.channel;
+                                if (videoFrontmatter.frontmatter.url) {
+                                    const extractedVideoId = extractVideoIdFromUrl(videoFrontmatter.frontmatter.url);
+                                    if (extractedVideoId)
+                                        videoId = extractedVideoId;
+                                }
+                            }
+                        }
+                    }
+                }
+                for (const parsedSegment of lensSection.segments) {
+                    const segmentResult = convertSegment(parsedSegment, lensSection, lensPath, files, sectionVisitedPaths, tierMap);
+                    errors.push(...segmentResult.errors);
+                    if (segmentResult.segment)
+                        segments.push(segmentResult.segment);
+                }
+                applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
+            }
+            const lo = loResult.learningOutcome;
+            items.push({
+                type: sectionType,
+                meta,
+                segments,
+                optional: section.fields.optional?.toLowerCase() === 'true' || lensRef.optional,
+                learningOutcomeId: lo.id ?? null,
+                learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
+                contentId: lens.id ?? null,
+                tldr: lens.tldr,
+                videoId: videoId ?? null,
+                ...computeSectionStats(segments),
+            });
+        }
+        // Process test section if present in this submodule
+        if (sub.test) {
+            const lo = loResult.learningOutcome;
+            const testSection = buildTestSection(sub.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+            if (testSection)
+                items.push(testSection);
+        }
+    }
+    // Also emit LO parse errors (we skipped earlier since we returned non-null)
+    errors.push(...loResult.errors);
+    return items;
 }
 /**
  * Flatten a Learning Outcome section by resolving its LO file and all referenced lenses.
@@ -277,6 +618,9 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
                             if (articleFrontmatter.frontmatter.source_url) {
                                 meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
                             }
+                            if (articleFrontmatter.frontmatter.published) {
+                                meta.published = String(articleFrontmatter.frontmatter.published);
+                            }
                         }
                     }
                 }
@@ -325,6 +669,8 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
                     segments.push(segmentResult.segment);
                 }
             }
+            // Apply collapsed content to article-excerpt segments
+            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
         }
         // Create a section for this lens
         // Optional can come from either:
@@ -338,42 +684,17 @@ function flattenLearningOutcomeSection(section, modulePath, files, visitedPaths,
             learningOutcomeId: lo.id ?? null,
             learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
             contentId: lens.id ?? null,
+            tldr: lens.tldr,
             videoId: videoId ?? null,
+            ...computeSectionStats(segments),
         };
         sections.push(resultSection);
     }
     // After lens processing, add test section if present with inline segments
-    if (lo.test && lo.test.segments.length > 0) {
-        const testSegments = [];
-        for (const parsedSegment of lo.test.segments) {
-            // For test sections with question/chat/text segments, no source file resolution needed
-            // Create a minimal stub lensSection since these segment types don't need it
-            const stubLensSection = {
-                type: 'page',
-                title: 'Test',
-                segments: [],
-                line: 0,
-            };
-            const segmentResult = convertSegment(parsedSegment, stubLensSection, loPath, files, visitedPaths, tierMap);
-            errors.push(...segmentResult.errors);
-            if (segmentResult.segment) {
-                testSegments.push(segmentResult.segment);
-            }
-        }
-        if (testSegments.length > 0) {
-            const hasFeedback = testSegments.some((s) => s.type === 'question' && s.feedback);
-            sections.push({
-                type: 'test',
-                meta: { title: 'Test' },
-                segments: testSegments,
-                optional: false,
-                ...(hasFeedback && { feedback: true }),
-                contentId: null,
-                learningOutcomeId: lo.id ?? null,
-                learningOutcomeName: loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null,
-                videoId: null,
-            });
-        }
+    if (lo.test) {
+        const testSection = buildTestSection(lo.test, lo.id ?? null, loPath.split('/').pop()?.replace(/\.md$/i, '') ?? null, loPath, files, visitedPaths, tierMap, errors);
+        if (testSection)
+            sections.push(testSection);
     }
     return { sections, errors };
 }
@@ -472,6 +793,9 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths, t
                             if (articleFrontmatter.frontmatter.source_url) {
                                 meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
                             }
+                            if (articleFrontmatter.frontmatter.published) {
+                                meta.published = String(articleFrontmatter.frontmatter.published);
+                            }
                         }
                     }
                 }
@@ -520,6 +844,8 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths, t
                     segments.push(segmentResult.segment);
                 }
             }
+            // Apply collapsed content to article-excerpt segments
+            applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
         }
         // Create a section for this lens
         const resultSection = {
@@ -530,7 +856,9 @@ function flattenUncategorizedSection(section, modulePath, files, visitedPaths, t
             learningOutcomeId: null,
             learningOutcomeName: null,
             contentId: lens.id ?? null,
+            tldr: lens.tldr,
             videoId: videoId ?? null,
+            ...computeSectionStats(segments),
         };
         sections.push(resultSection);
     }
@@ -646,6 +974,59 @@ function parseUncategorizedLensRefs(body, parentPath) {
     }
     return lensRefs;
 }
+/**
+ * After converting segments, apply collapsed_before/collapsed_after to article-excerpt segments.
+ * This resolves the article source once and calls bundleArticleWithCollapsed to compute
+ * which parts of the article are outside the excerpted ranges.
+ */
+function applyCollapsedContent(segments, parsedSegments, lensSection, lensPath, files) {
+    // Only process lens-article sections with article-excerpt segments
+    if (lensSection.type !== 'lens-article' || !lensSection.source)
+        return;
+    // Collect article-excerpt segment indices
+    const excerptIndices = [];
+    for (let i = 0; i < segments.length; i++) {
+        if (segments[i].type === 'article-excerpt') {
+            excerptIndices.push(i);
+        }
+    }
+    if (excerptIndices.length === 0)
+        return;
+    // Map to parsed segments to get anchors (order-preserving)
+    const parsedExcerpts = parsedSegments.filter(ps => ps.type === 'article-excerpt');
+    const excerptInfos = excerptIndices.map((segIdx, i) => ({
+        index: segIdx,
+        from: parsedExcerpts[i]?.fromAnchor,
+        to: parsedExcerpts[i]?.toAnchor,
+    }));
+    // Only apply collapsed if at least one excerpt has anchors
+    const hasAnchors = excerptInfos.some(e => e.from || e.to);
+    if (!hasAnchors)
+        return;
+    // Resolve article path
+    const wikilink = parseWikilink(lensSection.source);
+    if (!wikilink || wikilink.error)
+        return;
+    const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
+    const articlePath = findFileWithExtension(articlePathResolved, files);
+    if (!articlePath)
+        return;
+    const articleContent = files.get(articlePath);
+    // Call bundleArticleWithCollapsed
+    const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map(e => ({ from: e.from, to: e.to })), articlePath);
+    // Overlay collapsed_before/collapsed_after onto segments
+    for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
+        const segIdx = excerptInfos[i].index;
+        const collapsed = collapsedResults[i];
+        const segment = segments[segIdx];
+        if (collapsed.collapsed_before) {
+            segment.collapsed_before = collapsed.collapsed_before;
+        }
+        if (collapsed.collapsed_after) {
+            segment.collapsed_after = collapsed.collapsed_after;
+        }
+    }
+}
 /**
  * Convert a parsed lens segment into a final flattened segment.
  * For article-excerpt and video-excerpt, this involves extracting content from source files.
@@ -746,7 +1127,7 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             // Extract the excerpt
             const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
@@ -840,7 +1221,7 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
             // Extract the video excerpt
             const excerptResult = extractVideoExcerpt(transcriptContent, parsedSegment.fromTimeStr, parsedSegment.toTimeStr, videoPath, timestamps);
             if (excerptResult.error) {
-                errors.push(excerptResult.error);
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
@@ -873,6 +1254,23 @@ function convertSegment(parsedSegment, lensSection, lensPath, files, visitedPath
                 segment.feedback = true;
             return { segment, errors };
         }
+        case 'roleplay': {
+            const segment = {
+                type: 'roleplay',
+                id: parsedSegment.id,
+                content: parsedSegment.content,
+                aiInstructions: parsedSegment.aiInstructions,
+            };
+            if (parsedSegment.openingMessage)
+                segment.openingMessage = parsedSegment.openingMessage;
+            if (parsedSegment.assessmentInstructions)
+                segment.assessmentInstructions = parsedSegment.assessmentInstructions;
+            if (parsedSegment.optional)
+                segment.optional = true;
+            if (parsedSegment.feedback)
+                segment.feedback = true;
+            return { segment, errors };
+        }
         default:
             return { segment: null, errors };
     }
@@ -898,7 +1296,7 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
     const errors = [];
     // Skip ignored lenses
     if (tierMap?.get(lensPath) === 'ignored') {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const lensContent = files.get(lensPath);
     if (!lensContent) {
@@ -907,7 +1305,7 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
             message: `Lens file not found: ${lensPath}`,
             severity: 'error',
         });
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     // Use pre-parsed lens if provided, otherwise parse
     const lens = preParsedLens ?? (() => {
@@ -916,7 +1314,7 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
         return lensResult.lens;
     })();
     if (!lens) {
-        return { module: null, errors };
+        return { module: null, modules: [], errors };
     }
     const visitedPaths = new Set([lensPath]);
     // Process lens sections into a single flattened Section
@@ -942,6 +1340,8 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
                             meta.author = articleFrontmatter.frontmatter.author;
                         if (articleFrontmatter.frontmatter.source_url)
                             meta.sourceUrl = articleFrontmatter.frontmatter.source_url;
+                        if (articleFrontmatter.frontmatter.published)
+                            meta.published = String(articleFrontmatter.frontmatter.published);
                     }
                 }
             }
@@ -982,6 +1382,8 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
             if (segmentResult.segment)
                 segments.push(segmentResult.segment);
         }
+        // Apply collapsed content to article-excerpt segments
+        applyCollapsedContent(segments, lensSection.segments, lensSection, lensPath, files);
     }
     // Fallback title from filename if not extracted from source metadata
     if (!meta.title) {
@@ -995,7 +1397,9 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
         learningOutcomeId: null,
         learningOutcomeName: null,
         contentId: lens.id ?? null,
+        tldr: lens.tldr,
         videoId: videoId ?? null,
+        ...computeSectionStats(segments),
     };
     const flattenedModule = {
         slug: 'lens/' + fileNameToSlug(lensPath),
@@ -1003,6 +1407,6 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
         contentId: lens.id ?? null,
         sections: [section],
     };
-    return { module: flattenedModule, errors };
+    return { module: flattenedModule, modules: [flattenedModule], errors };
 }
 //# sourceMappingURL=index.js.map