lens-content-processor 0.33.0 → 0.35.0

package/dist/flattener/index.js

@@ -5,9 +5,10 @@ import { parseLearningOutcome, } from "../parser/learning-outcome.js";
 import { parseLens, } from "../parser/lens.js";
 import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, isSoleWikilink, findWikilinks, findBrokenPromptImports, } from "../parser/wikilink.js";
 import { parsePrompt, isPromptPath } from "../parser/prompt.js";
+import { toFlattenedResponseSegment } from "../parser/response-segments.js";
 import { parseFrontmatter } from "../parser/frontmatter.js";
 import { fileNameToSlug } from "../utils/slug.js";
-import { extractArticleExcerpt, bundleArticleWithCollapsed, } from "../bundler/article.js";
+import { extractArticleExcerpt, resolveArticleSource, } from "../bundler/article.js";
 import { extractVideoExcerpt, resolveVideoRange, } from "../bundler/video.js";
 import { attachSourceLine, getSourceLine } from "../source-location.js";
 function countWords(text) {
@@ -85,15 +86,21 @@ function lensSectionFields(lens, segments, file, line, errors) {
         contentId: lens.id ?? null,
         tldr: lens.tldr,
         summaryForTutor: lens.summaryForTutor,
-        minChatMessages: lens.minChatMessages,
-        durationMinutes: lens.durationMinutes,
-        readingMinutes: lens.readingMinutes,
-        tutorMinutes: lens.tutorMinutes,
-        addToAiContext: lens.addToAiContext,
+        ...definedProperties({
+            minChatMessages: lens.minChatMessages,
+            durationMinutes: lens.durationMinutes,
+            readingMinutes: lens.readingMinutes,
+            tutorMinutes: lens.tutorMinutes,
+            addToAiContext: lens.addToAiContext,
+            displayType: computeDisplayType(segments),
+        }),
         ...stats,
-        displayType: computeDisplayType(segments),
     };
 }
+/** Keep optional output fields absent instead of materializing `undefined`. */
+function definedProperties(value) {
+    return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
+}
 /** Compute word count and video duration from a section's segments. */
 function computeSectionStats(segments) {
     let words = 0;
@@ -280,15 +287,24 @@ export function splitAtBoundaries(items, parentSlug, parentTitle) {
 function flattenSingleLens(lens, lensPath, files, visitedPaths, tierMap) {
     const errors = [];
     const segments = [];
-    for (const parsedSegment of lens.segments) {
+    // Article segments are resolved up front, grouped per source file (one
+    // parsed segment can expand to several output segments when the article
+    // declares inline videos).
+    const articlePlan = planLensArticles(lens.segments, lensPath, files, visitedPaths, tierMap, { collapsed: true });
+    for (let i = 0; i < lens.segments.length; i++) {
+        const parsedSegment = lens.segments[i];
+        const planned = articlePlan.get(i);
+        if (planned) {
+            errors.push(...planned.errors);
+            segments.push(...planned.segments);
+            continue;
+        }
         const segmentResult = convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap);
         errors.push(...segmentResult.errors);
         if (segmentResult.segment) {
             segments.push(segmentResult.segment);
         }
     }
-    // Apply collapsed content for article segments
-    applyCollapsedContent(segments, lens.segments, lensPath, files);
     return { segments, errors };
 }
 // ─── Module flattening ───
@@ -505,11 +521,11 @@ export function flattenModule(modulePath, files, visitedPaths = new Set(), tierM
         // Module-level add_to_ai_context, with any submodule-level entries merged in
         // (module first, deduped). When the submodule has no field, this is exactly
         // the module's own value.
-        ...(g.addToAiContext !== undefined
-            ? {
-                addToAiContext: mergeAiContext(parsedModule.addToAiContext, g.addToAiContext),
-            }
-            : { addToAiContext: parsedModule.addToAiContext }),
+        ...definedProperties({
+            addToAiContext: g.addToAiContext !== undefined
+                ? mergeAiContext(parsedModule.addToAiContext, g.addToAiContext)
+                : parsedModule.addToAiContext,
+        }),
     }));
     // Enrich ::card links in text segments with computed metadata (duration, attribution, displayType)
     for (const mod of resultModules) {
@@ -532,7 +548,16 @@ function buildTestSection(testRef, loId, loName, loPath, files, visitedPaths, ti
     if (!testRef.segments.length)
         return null;
     const testSegments = [];
-    for (const parsedSegment of testRef.segments) {
+    // collapsed: false — test sections never carried collapsed_* fields
+    const articlePlan = planLensArticles(testRef.segments, loPath, files, new Set(visitedPaths), tierMap, { collapsed: false });
+    for (let i = 0; i < testRef.segments.length; i++) {
+        const parsedSegment = testRef.segments[i];
+        const planned = articlePlan.get(i);
+        if (planned) {
+            errors.push(...planned.errors);
+            testSegments.push(...planned.segments);
+            continue;
+        }
         const segmentResult = convertSegment(parsedSegment, loPath, files, new Set(visitedPaths), tierMap);
         errors.push(...segmentResult.errors);
         if (segmentResult.segment) {
@@ -786,69 +811,312 @@ function flattenLensSection(section, modulePath, files, visitedPaths, tierMap, i
     sections.push(resultSection);
     return { sections, errors };
 }
-// ─── Collapsed content and segment conversion ───
 /**
- * After converting segments, apply collapsed_before/collapsed_after to article segments.
- * Groups article segments by their resolved source path, then calls bundleArticleWithCollapsed
- * for each group.
+ * Resolve a video source wikilink into a VideoSegment: transcript lookup,
+ * tier/circular checks, `.timestamps.json` sidecar, from/to range resolution,
+ * and transcript excerpt extraction. Shared by lens `#### Video` segments and
+ * inline `::video` article imports — only error attribution differs (ctx).
+ */
+function resolveVideoSegment(source, fromTimeStr, toTimeStr, ctx) {
+    const errors = [];
+    const { files, visitedPaths, tierMap } = ctx;
+    const withLine = (error) => ctx.errorLine !== undefined ? { ...error, line: ctx.errorLine } : error;
+    const wikilink = parseWikilink(source);
+    if (!wikilink || wikilink.error) {
+        const suggestion = wikilink?.correctedPath
+            ? `Did you mean '[[${wikilink.correctedPath}]]'?`
+            : undefined;
+        errors.push(withLine({
+            file: ctx.errorFile,
+            message: `Invalid wikilink in video source: ${source}`,
+            suggestion,
+            severity: "error",
+        }));
+        return { segment: null, errors };
+    }
+    const videoPathResolved = resolveWikilinkPath(wikilink.path, ctx.resolveFrom);
+    const videoPath = findFileWithExtension(videoPathResolved, files);
+    if (!videoPath) {
+        const similarFiles = findSimilarFiles(videoPathResolved, files, "video_transcripts");
+        const suggestion = formatSuggestion(similarFiles, ctx.errorFile) ??
+            "Check the file path in the wiki-link";
+        const error = withLine({
+            file: ctx.errorFile,
+            message: `Referenced video transcript file not found: ${videoPathResolved}`,
+            suggestion,
+            severity: "error",
+        });
+        if (ctx.notFoundCode)
+            error.code = ctx.notFoundCode;
+        errors.push(error);
+        return { segment: null, errors };
+    }
+    // Check tier violation (referencing file -> video transcript)
+    if (tierMap) {
+        const parentTier = tierMap.get(ctx.resolveFrom) ?? "production";
+        const childTier = tierMap.get(videoPath) ?? "production";
+        const violation = checkTierViolation(ctx.resolveFrom, parentTier, videoPath, childTier, "video transcript");
+        if (violation) {
+            errors.push(violation);
+        }
+        if (childTier === "ignored") {
+            return { segment: null, errors };
+        }
+    }
+    if (visitedPaths.has(videoPath)) {
+        errors.push(withLine({
+            file: ctx.errorFile,
+            message: `Circular reference detected: ${videoPath}`,
+            severity: "error",
+        }));
+        return { segment: null, errors };
+    }
+    const transcriptContent = files.get(videoPath);
+    // Extract video metadata from frontmatter
+    const videoFrontmatter = parseFrontmatter(transcriptContent, videoPath);
+    const vfm = videoFrontmatter.frontmatter;
+    // Look for corresponding .timestamps.json file
+    const timestampsPath = videoPath.replace(/\.md$/, ".timestamps.json");
+    let timestamps;
+    if (files.has(timestampsPath)) {
+        try {
+            timestamps = JSON.parse(files.get(timestampsPath));
+        }
+        catch {
+            // JSON parse error - will fall back to inline timestamps
+        }
+    }
+    // Extract the video excerpt. Caller from/to (lens fields or ::video attrs)
+    // win over the transcript's frontmatter defaults (which win over
+    // 0:00/open-ended).
+    const { fromTime, toTime } = resolveVideoRange(fromTimeStr, toTimeStr, vfm);
+    const excerptResult = extractVideoExcerpt(transcriptContent, fromTime, toTime, videoPath, timestamps);
+    if (excerptResult.error) {
+        const error = { ...excerptResult.error, file: ctx.errorFile };
+        const line = ctx.errorLine ?? ctx.excerptErrorLine;
+        if (line !== undefined)
+            error.line = line;
+        errors.push(error);
+        return { segment: null, errors };
+    }
+    const segment = {
+        type: "video",
+        from: excerptResult.from,
+        to: excerptResult.to,
+        transcript: excerptResult.transcript,
+    };
+    // Populate metadata from video transcript frontmatter
+    if (vfm.title)
+        segment.title = vfm.title;
+    if (vfm.channel)
+        segment.channel = vfm.channel;
+    if (vfm.url) {
+        const extracted = extractVideoFromUrl(vfm.url);
+        if (extracted) {
+            segment.videoId = extracted.videoId;
+            if (extracted.isShort)
+                segment.isShort = true;
+        }
+    }
+    segment.sourcePath = videoPath;
+    return { segment, errors };
+}
+/** 1-based line of an inline import's source line in the raw article file. */
+function findImportLine(rawContent, rawLine) {
+    const lines = rawContent.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        if (lines[i].trim() === rawLine)
+            return i + 1;
+    }
+    return undefined;
+}
+/**
+ * Resolve every article segment of a lens (or LO test) up front, grouped by
+ * source file so each source is read once: excerpt extraction, collapsed
+ * (elided) text at excerpt boundaries, and splitting around inline `::video`
+ * imports all happen in resolveArticleSource. The result is keyed by the
+ * parsed segment's index, so one parsed article segment can expand to several
+ * output segments (article part / video / article part) without any
+ * positional re-pairing.
+ *
+ * `collapsed: false` (LO test sections) skips collapsed enrichment, matching
+ * the previous pipeline where only lens sections received collapsed fields.
  */
-function applyCollapsedContent(segments, parsedSegments, lensPath, files) {
-    // Build groups of article segments by resolved source path
-    const sourceGroups = new Map();
-    // Map output article segments back to parsed segments for source info
-    let articleOutputIdx = 0;
+function planLensArticles(parsedSegments, lensPath, files, visitedPaths, tierMap, options) {
+    const plan = new Map();
+    const groups = new Map();
     for (let i = 0; i < parsedSegments.length; i++) {
-        const ps = parsedSegments[i];
-        if (ps.type !== "article")
+        const parsedSegment = parsedSegments[i];
+        if (parsedSegment.type !== "article")
+            continue;
+        const entry = { segments: [], errors: [] };
+        plan.set(i, entry);
+        // Source comes from the parsed segment (set via inheritance in parseLens)
+        if (!parsedSegment.source) {
+            entry.errors.push({
+                file: lensPath,
+                message: "Article segment missing source (should have been set via inheritance)",
+                severity: "error",
+            });
             continue;
-        // Find corresponding output segment
-        while (articleOutputIdx < segments.length &&
-            segments[articleOutputIdx].type !== "article") {
-            articleOutputIdx++;
         }
-        if (articleOutputIdx >= segments.length)
-            break;
-        if (ps.source) {
-            const wikilink = parseWikilink(ps.source);
-            if (wikilink && !wikilink.error) {
-                const resolved = resolveWikilinkPath(wikilink.path, lensPath);
-                const actualPath = findFileWithExtension(resolved, files);
-                if (actualPath) {
-                    if (!sourceGroups.has(actualPath)) {
-                        sourceGroups.set(actualPath, []);
-                    }
-                    sourceGroups.get(actualPath).push({
-                        segmentIndex: articleOutputIdx,
-                        fromAnchor: ps.fromAnchor,
-                        toAnchor: ps.toAnchor,
-                    });
-                }
+        const wikilink = parseWikilink(parsedSegment.source);
+        if (!wikilink || wikilink.error) {
+            const suggestion = wikilink?.correctedPath
+                ? `Did you mean '[[${wikilink.correctedPath}]]'?`
+                : undefined;
+            entry.errors.push({
+                file: lensPath,
+                message: `Invalid wikilink in article source: ${parsedSegment.source}`,
+                suggestion,
+                severity: "error",
+            });
+            continue;
+        }
+        const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
+        const articlePath = findFileWithExtension(articlePathResolved, files);
+        if (!articlePath) {
+            const similarFiles = findSimilarFiles(articlePathResolved, files, "articles");
+            const suggestion = formatSuggestion(similarFiles, lensPath) ??
+                "Check the file path in the wiki-link";
+            entry.errors.push({
+                file: lensPath,
+                message: `Referenced article file not found: ${articlePathResolved}`,
+                suggestion,
+                severity: "error",
+            });
+            continue;
+        }
+        // Check tier violation (Lens -> Article)
+        if (tierMap) {
+            const parentTier = tierMap.get(lensPath) ?? "production";
+            const childTier = tierMap.get(articlePath) ?? "production";
+            const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, "article");
+            if (violation) {
+                entry.errors.push(violation);
+            }
+            if (childTier === "ignored") {
+                continue;
             }
         }
-        articleOutputIdx++;
-    }
-    // For each source group, apply collapsed content
-    for (const [articlePath, excerptInfos] of sourceGroups) {
-        // Only apply collapsed if at least one excerpt has anchors
-        const hasAnchors = excerptInfos.some((e) => e.fromAnchor || e.toAnchor);
-        if (!hasAnchors)
+        if (visitedPaths.has(articlePath)) {
+            entry.errors.push({
+                file: lensPath,
+                message: `Circular reference detected: ${articlePath}`,
+                severity: "error",
+            });
             continue;
+        }
+        let group = groups.get(articlePath);
+        if (!group) {
+            group = { indices: [], segments: [] };
+            groups.set(articlePath, group);
+        }
+        group.indices.push(i);
+        group.segments.push(parsedSegment);
+    }
+    for (const [articlePath, group] of groups) {
         const articleContent = files.get(articlePath);
-        if (!articleContent)
-            continue;
-        const collapsedResults = bundleArticleWithCollapsed(articleContent, excerptInfos.map((e) => ({ from: e.fromAnchor, to: e.toAnchor })), articlePath);
-        for (let i = 0; i < excerptInfos.length && i < collapsedResults.length; i++) {
-            const segIdx = excerptInfos[i].segmentIndex;
-            const collapsed = collapsedResults[i];
-            const segment = segments[segIdx];
-            if (collapsed.collapsed_before) {
-                segment.collapsed_before = collapsed.collapsed_before;
+        const fm = parseFrontmatter(articleContent, articlePath).frontmatter;
+        const resolved = resolveArticleSource(articleContent, group.segments.map((ps) => ({ from: ps.fromAnchor, to: ps.toAnchor })), articlePath);
+        for (let k = 0; k < group.segments.length; k++) {
+            const parsedSegment = group.segments[k];
+            const entry = plan.get(group.indices[k]);
+            const excerpt = resolved[k];
+            if (excerpt.error) {
+                entry.errors.push({
+                    ...excerpt.error,
+                    file: lensPath,
+                    line: getSourceLine(parsedSegment),
+                });
+                continue;
+            }
+            const makeArticleSegment = (content) => {
+                const segment = { type: "article", content };
+                if (fm.title)
+                    segment.title = fm.title;
+                if (fm.author) {
+                    const raw = fm.author;
+                    segment.author = Array.isArray(raw) ? raw.join(", ") : String(raw);
+                }
+                if (fm.source_url)
+                    segment.sourceUrl = fm.source_url;
+                if (fm.published)
+                    segment.published = String(fm.published);
+                segment.sourcePath = articlePath;
+                if (parsedSegment.optional)
+                    segment.optional = true;
+                return attachSourceLine(segment, getSourceLine(parsedSegment));
+            };
+            const applyCollapsed = (segments) => {
+                if (!options.collapsed)
+                    return;
+                // collapsed_* describe the excerpt's outer boundaries, so they go on
+                // the first/last article part; a video at the very edge has no prose
+                // to carry them, and the (rare) all-video excerpt drops them.
+                const articleParts = segments.filter((s) => s.type === "article");
+                if (articleParts.length === 0)
+                    return;
+                if (excerpt.collapsed_before) {
+                    articleParts[0].collapsed_before = excerpt.collapsed_before;
+                }
+                if (excerpt.collapsed_after) {
+                    articleParts[articleParts.length - 1].collapsed_after =
+                        excerpt.collapsed_after;
+                }
+            };
+            if (!excerpt.parts) {
+                const segment = makeArticleSegment(excerpt.content);
+                applyCollapsed([segment]);
+                entry.segments.push(segment);
+                continue;
             }
-            if (collapsed.collapsed_after) {
-                segment.collapsed_after = collapsed.collapsed_after;
+            // The excerpt contains inline videos: emit article/video/article parts.
+            const partSegments = [];
+            let videoFailed = false;
+            for (const part of excerpt.parts) {
+                if (part.kind === "prose") {
+                    const segment = makeArticleSegment(part.content);
+                    if (part.contiguousWithPrev)
+                        segment.skipStartMarker = true;
+                    if (part.contiguousWithNext)
+                        segment.skipEndMarker = true;
+                    partSegments.push(segment);
+                    continue;
+                }
+                const videoResult = resolveVideoSegment(`[[${part.target}]]`, part.from, part.to, {
+                    resolveFrom: articlePath,
+                    errorFile: articlePath,
+                    errorLine: findImportLine(articleContent, part.rawLine),
+                    notFoundCode: "article.video-import-not-found",
+                    files,
+                    visitedPaths,
+                    tierMap,
+                });
+                entry.errors.push(...videoResult.errors);
+                if (!videoResult.segment) {
+                    videoFailed = true;
+                    continue;
+                }
+                videoResult.segment.partOfArticle = true;
+                if (parsedSegment.optional)
+                    videoResult.segment.optional = true;
+                partSegments.push(attachSourceLine(videoResult.segment, getSourceLine(parsedSegment)));
+            }
+            if (videoFailed) {
+                // A broken import must not leave a half-split article with dangling
+                // seam flags: fall back to the un-split excerpt (error already
+                // recorded above).
+                const segment = makeArticleSegment(excerpt.content);
+                applyCollapsed([segment]);
+                entry.segments.push(segment);
+                continue;
             }
+            applyCollapsed(partSegments);
+            entry.segments.push(...partSegments);
         }
     }
+    return plan;
 }
 /**
  * Resolve a single prompt-import wikilink into the referenced prompt file's
@@ -938,7 +1206,7 @@ function resolvePromptImport(value, fieldName, lensPath, files, tierMap) {
  *   doesn't resolve is an error, and the segment is dropped like a failed
  *   sole import.
  */
-function expandPromptImports(value, fieldName, lensPath, files, tierMap) {
+export function expandPromptImports(value, fieldName, lensPath, files, tierMap) {
     // Sole wikilink — the whole field is the import (including recognizably
     // malformed single wikilinks, which must fail loudly).
     if (isSoleWikilink(value)) {
@@ -1041,101 +1309,11 @@ function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
                 errors,
             };
         }
-        case "article": {
-            // Source comes from the parsed segment (set via inheritance in parseLens)
-            if (!parsedSegment.source) {
-                errors.push({
-                    file: lensPath,
-                    message: "Article segment missing source (should have been set via inheritance)",
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            const wikilink = parseWikilink(parsedSegment.source);
-            if (!wikilink || wikilink.error) {
-                const suggestion = wikilink?.correctedPath
-                    ? `Did you mean '[[${wikilink.correctedPath}]]'?`
-                    : undefined;
-                errors.push({
-                    file: lensPath,
-                    message: `Invalid wikilink in article source: ${parsedSegment.source}`,
-                    suggestion,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            const articlePathResolved = resolveWikilinkPath(wikilink.path, lensPath);
-            const articlePath = findFileWithExtension(articlePathResolved, files);
-            if (!articlePath) {
-                const similarFiles = findSimilarFiles(articlePathResolved, files, "articles");
-                const suggestion = formatSuggestion(similarFiles, lensPath) ??
-                    "Check the file path in the wiki-link";
-                errors.push({
-                    file: lensPath,
-                    message: `Referenced article file not found: ${articlePathResolved}`,
-                    suggestion,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            // Check tier violation (Lens -> Article)
-            if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? "production";
-                const childTier = tierMap.get(articlePath) ?? "production";
-                const violation = checkTierViolation(lensPath, parentTier, articlePath, childTier, "article");
-                if (violation) {
-                    errors.push(violation);
-                }
-                if (childTier === "ignored") {
-                    return { segment: null, errors };
-                }
-            }
-            if (visitedPaths.has(articlePath)) {
-                errors.push({
-                    file: lensPath,
-                    message: `Circular reference detected: ${articlePath}`,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            const articleContent = files.get(articlePath);
-            // Extract article metadata from frontmatter
-            const articleFrontmatter = parseFrontmatter(articleContent, articlePath);
-            const fm = articleFrontmatter.frontmatter;
-            // Extract the excerpt
-            const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
-            if (excerptResult.error) {
-                errors.push({
-                    ...excerptResult.error,
-                    file: lensPath,
-                    line: getSourceLine(parsedSegment),
-                });
-                return { segment: null, errors };
-            }
-            const segment = {
-                type: "article",
-                content: excerptResult.content,
-            };
-            // Populate metadata from article frontmatter
-            if (fm.title)
-                segment.title = fm.title;
-            if (fm.author) {
-                const raw = fm.author;
-                segment.author = Array.isArray(raw) ? raw.join(", ") : String(raw);
-            }
-            if (fm.source_url)
-                segment.sourceUrl = fm.source_url;
-            if (fm.published)
-                segment.published = String(fm.published);
-            segment.sourcePath = articlePath;
-            if (parsedSegment.optional) {
-                segment.optional = true;
-            }
-            return {
-                segment: attachSourceLine(segment, getSourceLine(parsedSegment)),
-                errors,
-            };
-        }
+        case "article":
+            // Article segments never reach convertSegment: planLensArticles
+            // resolves them, grouped per source file, so excerpts, collapsed text,
+            // and inline-video splits are computed in one pass.
+            return { segment: null, errors };
         case "video": {
             // Source comes from the parsed segment (set via inheritance in parseLens)
             if (!parsedSegment.source) {
@@ -1146,104 +1324,23 @@ function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
                 });
                 return { segment: null, errors };
             }
-            const wikilink = parseWikilink(parsedSegment.source);
-            if (!wikilink || wikilink.error) {
-                const suggestion = wikilink?.correctedPath
-                    ? `Did you mean '[[${wikilink.correctedPath}]]'?`
-                    : undefined;
-                errors.push({
-                    file: lensPath,
-                    message: `Invalid wikilink in video source: ${parsedSegment.source}`,
-                    suggestion,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            const videoPathResolved = resolveWikilinkPath(wikilink.path, lensPath);
-            const videoPath = findFileWithExtension(videoPathResolved, files);
-            if (!videoPath) {
-                const similarFiles = findSimilarFiles(videoPathResolved, files, "video_transcripts");
-                const suggestion = formatSuggestion(similarFiles, lensPath) ??
-                    "Check the file path in the wiki-link";
-                errors.push({
-                    file: lensPath,
-                    message: `Referenced video transcript file not found: ${videoPathResolved}`,
-                    suggestion,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            // Check tier violation (Lens -> Video)
-            if (tierMap) {
-                const parentTier = tierMap.get(lensPath) ?? "production";
-                const childTier = tierMap.get(videoPath) ?? "production";
-                const violation = checkTierViolation(lensPath, parentTier, videoPath, childTier, "video transcript");
-                if (violation) {
-                    errors.push(violation);
-                }
-                if (childTier === "ignored") {
-                    return { segment: null, errors };
-                }
-            }
-            if (visitedPaths.has(videoPath)) {
-                errors.push({
-                    file: lensPath,
-                    message: `Circular reference detected: ${videoPath}`,
-                    severity: "error",
-                });
-                return { segment: null, errors };
-            }
-            const transcriptContent = files.get(videoPath);
-            // Extract video metadata from frontmatter
-            const videoFrontmatter = parseFrontmatter(transcriptContent, videoPath);
-            const vfm = videoFrontmatter.frontmatter;
-            // Look for corresponding .timestamps.json file
-            const timestampsPath = videoPath.replace(/\.md$/, ".timestamps.json");
-            let timestamps;
-            if (files.has(timestampsPath)) {
-                try {
-                    timestamps = JSON.parse(files.get(timestampsPath));
-                }
-                catch {
-                    // JSON parse error - will fall back to inline timestamps
-                }
-            }
-            // Extract the video excerpt. Lens from::/to:: win over the
-            // transcript's frontmatter defaults (which win over 0:00/open-ended).
-            const { fromTime, toTime } = resolveVideoRange(parsedSegment.fromTimeStr, parsedSegment.toTimeStr, vfm);
-            const excerptResult = extractVideoExcerpt(transcriptContent, fromTime, toTime, videoPath, timestamps);
-            if (excerptResult.error) {
-                errors.push({
-                    ...excerptResult.error,
-                    file: lensPath,
-                    line: getSourceLine(parsedSegment),
-                });
+            const result = resolveVideoSegment(parsedSegment.source, parsedSegment.fromTimeStr, parsedSegment.toTimeStr, {
+                resolveFrom: lensPath,
+                errorFile: lensPath,
+                excerptErrorLine: getSourceLine(parsedSegment),
+                files,
+                visitedPaths,
+                tierMap,
+            });
+            errors.push(...result.errors);
+            if (!result.segment) {
                 return { segment: null, errors };
             }
-            const segment = {
-                type: "video",
-                from: excerptResult.from,
-                to: excerptResult.to,
-                transcript: excerptResult.transcript,
-            };
-            // Populate metadata from video transcript frontmatter
-            if (vfm.title)
-                segment.title = vfm.title;
-            if (vfm.channel)
-                segment.channel = vfm.channel;
-            if (vfm.url) {
-                const extracted = extractVideoFromUrl(vfm.url);
-                if (extracted) {
-                    segment.videoId = extracted.videoId;
-                    if (extracted.isShort)
-                        segment.isShort = true;
-                }
-            }
             if (parsedSegment.optional) {
-                segment.optional = true;
+                result.segment.optional = true;
             }
             return {
-                segment: attachSourceLine(segment, getSourceLine(parsedSegment)),
+                segment: attachSourceLine(result.segment, getSourceLine(parsedSegment)),
                 errors,
             };
         }
@@ -1257,6 +1354,30 @@ function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
                 }
                 assessmentInstructions = resolved.value;
             }
+            // New-style response segment (`#### Question: <Subtype>`): pass the
+            // parsed shape through, with prompt imports expanded in both
+            // instruction fields. Answers (correctOptions, blank expectations)
+            // stay in the cache JSON — the web API strips them before serving.
+            if ("questionType" in parsedSegment) {
+                let feedbackInstructions = parsedSegment.feedbackInstructions;
+                if (feedbackInstructions) {
+                    const resolved = expandPromptImports(feedbackInstructions, "feedback-instructions", lensPath, files, tierMap);
+                    errors.push(...resolved.errors);
+                    if (resolved.value === null) {
+                        return { segment: null, errors };
+                    }
+                    feedbackInstructions = resolved.value;
+                }
+                const segment = toFlattenedResponseSegment(parsedSegment);
+                if (assessmentInstructions)
+                    segment.assessmentInstructions = assessmentInstructions;
+                if (feedbackInstructions)
+                    segment.feedbackInstructions = feedbackInstructions;
+                return {
+                    segment: attachSourceLine(segment, getSourceLine(parsedSegment)),
+                    errors,
+                };
+            }
             const segment = {
                 type: "question",
                 content: parsedSegment.content,
@@ -1389,11 +1510,7 @@ function convertSegment(parsedSegment, lensPath, files, visitedPaths, tierMap) {
             }
             const excerptResult = extractArticleExcerpt(articleContent, parsedSegment.fromAnchor, parsedSegment.toAnchor, articlePath);
             if (excerptResult.error) {
-                errors.push({
-                    ...excerptResult.error,
-                    file: lensPath,
-                    line: getSourceLine(parsedSegment),
-                });
+                errors.push({ ...excerptResult.error, file: lensPath });
                 return { segment: null, errors };
             }
             const segment = {
@@ -1484,13 +1601,13 @@ export function flattenLens(lensPath, files, tierMap, preParsedLens) {
                 .replace(/\b\w/g, (c) => c.toUpperCase());
         }
     }
-    const section = attachSourceLine({
+    const section = {
         type: "lens",
         meta,
         sourcePath: lensPath,
         optional: false,
         ...lensSectionFields(lens, segments, lensPath, 1, errors),
-    }, getSourceLine(lens));
+    };
     const flattenedModule = {
         slug: "lens/" + fileNameToSlug(lensPath),
         title: meta.title ?? fileNameToSlug(lensPath),