lens-content-processor 0.78.0 → 0.80.0

package/dist/dropped.js

@@ -0,0 +1,253 @@
+// What errors left out of pages: the "dropped" artifact behind the error
+// boxes that admins see on staging. It is side data: no page, error or other
+// output changes because of it.
+//
+// Each place that leaves content out because of an error records it, with
+// the error that dropped it:
+// - a section that could not be built stays in its module as a placeholder
+//   (droppedSection), so the module's own ordering (tests to the end of their
+//   submodule, module-level tests to "Test Your Understanding") puts it where
+//   the section would have been; so does a section header that the parser
+//   refused. Every decision about sections skips the placeholders, and
+//   placeDropped takes them out before the pages are made;
+// - a segment that could not be built rides on its section
+//   (attachDroppedSegments); one that the parser refused rides on the parsed
+//   lens or test until its section is made. Its item follows that section;
+// - a page that could not be built at all is a moduleError entry.
+// crossCheck is the check that is not a hand-kept list: a section that a
+// module declares, that none of its pages holds, and that no record
+// explains, becomes an item too.
+//
+// The entries travel in their path unit's contribution, so an incremental
+// job keeps them exactly as current as the pages. droppedArtifact swaps each
+// recorded error for its copy in the final error list (same dedupe key),
+// which carries the category.
+import { errorKey } from "./dedupe-errors.js";
+import { parseFrontmatter } from "./parser/frontmatter.js";
+import { parseWikilink, resolveWikilinkPath } from "./parser/wikilink.js";
+/** A record placed after `sections[index]` (-1: before the first). */
+export function placedAfter(index, sections, drop) {
+    const afterContentId = sections[index]?.contentId ?? null;
+    return { afterSection: index, afterContentId, ...drop };
+}
+export function droppedContent(kind, title, sourcePath, error) {
+    return { what: { kind, title, sourcePath }, error };
+}
+/**
+ * The error that made a step return nothing: the first error of severity
+ * "error" it reported, else its first error. Undefined when it reported none,
+ * which leaves nothing to record.
+ */
+export function dropCause(errors) {
+    return errors.find((error) => error.severity === "error") ?? errors[0];
+}
+/** A file's name without its folder and `.md`: the title of what it held when nothing better is known. */
+export function fileStem(path) {
+    return path.slice(path.lastIndexOf("/") + 1).replace(/\.md$/i, "");
+}
+// Symbol-keyed and not enumerable, like source lines (source-location.ts):
+// JSON and the build layer's codec never see them, so pages are unchanged.
+const PLACEHOLDER = Symbol("lens-content-dropped-section");
+const SEGMENTS = Symbol("lens-content-dropped-segments");
+/**
+ * A section that an error dropped, standing where the section would have
+ * been. It has the section's type, so ordering by type moves it as it would
+ * have moved the section, and no sourcePath, so outcome tagging skips it.
+ */
+export function droppedSection(type, drop) {
+    const placeholder = {
+        type,
+        meta: {},
+        segments: [],
+        contentId: null,
+        learningOutcomes: [],
+    };
+    return Object.defineProperty(placeholder, PLACEHOLDER, { value: drop });
+}
+/** True for anything but a droppedSection placeholder. */
+export function isBuilt(item) {
+    return !(PLACEHOLDER in item);
+}
+/** Record segments that errors dropped from `section` (or from a parsed lens or test, before it has a section). */
+export function attachDroppedSegments(section, drops) {
+    if (drops.length === 0)
+        return section;
+    return Object.defineProperty(section, SEGMENTS, {
+        value: [...droppedSegmentsOf(section), ...drops],
+        configurable: true,
+    });
+}
+/** The segment records attached to `value`: a section, or a parsed lens or test whose parser refused some. */
+export function droppedSegmentsOf(value) {
+    return value[SEGMENTS] ?? [];
+}
+/**
+ * The record of a segment that a step returned no segment for, with the
+ * error that dropped it (none when the step reported no error). A segment
+ * with a source is named after its file; one without, after its type.
+ */
+export function droppedSegment(type, source, path, errors) {
+    const cause = dropCause(errors);
+    if (!cause)
+        return [];
+    const target = source && parseWikilink(source, path);
+    const sourcePath = target && !target.error && target.path
+        ? resolveWikilinkPath(target.path, path)
+        : path;
+    const title = sourcePath !== path
+        ? fileStem(sourcePath)
+        : type[0].toUpperCase() + type.slice(1);
+    return [droppedContent("segment", title, sourcePath, cause)];
+}
+/**
+ * Take the placeholders out of a page's sections and place every record: a
+ * placeholder follows the section before it, a dropped segment follows its
+ * own section.
+ */
+export function placeDropped(sections) {
+    const kept = [];
+    const items = [];
+    const place = (drop) => items.push(placedAfter(kept.length - 1, kept, drop));
+    for (const section of sections) {
+        const drop = section[PLACEHOLDER];
+        if (drop) {
+            place(drop);
+            continue;
+        }
+        kept.push(section);
+        droppedSegmentsOf(section).forEach(place);
+    }
+    return { sections: kept, items };
+}
+/**
+ * The check that does not depend on every drop site recording itself, as a
+ * site added later may not. A section that the module declares, that none of
+ * its pages holds, and that no record explains, is matched by sourcePath and
+ * recorded after the last section of its page. Its error says that no error
+ * explains it; it is not in the error list, which stays as it is. A header
+ * the parser refused (`refused`, SectionsResult.refused) declares nothing, so
+ * its record, known by its error, explains no declared section.
+ */
+export function crossCheck(modulePath, pages, items, declared, refused) {
+    const unmatched = new Map();
+    const add = (path) => {
+        if (path !== undefined)
+            unmatched.set(path, (unmatched.get(path) ?? 0) + 1);
+    };
+    for (const page of pages)
+        page.sections.forEach((s) => add(s.sourcePath));
+    const refusedErrors = new Set(refused.map(({ error }) => errorKey(error)));
+    for (const { what, error } of items.flat()) {
+        if (what.kind === "section" && !refusedErrors.has(errorKey(error)))
+            add(what.sourcePath);
+    }
+    for (const section of declared) {
+        const left = unmatched.get(section.sourcePath) ?? 0;
+        if (left > 0) {
+            unmatched.set(section.sourcePath, left - 1);
+            continue;
+        }
+        const page = Math.min(section.page, pages.length - 1);
+        const { sections } = pages[page];
+        const error = {
+            file: modulePath,
+            line: section.line,
+            code: "validator.section-dropped",
+            message: "This section is missing from its page, and no error says why",
+            suggestion: "Tell a developer: the validator left content out without an error",
+            severity: "error",
+        };
+        items[page].push(placedAfter(sections.length - 1, sections, droppedContent("section", section.title, section.sourcePath, error)));
+    }
+}
+/** The entries of a file's pages that lost something, in page order. */
+export function pageEntries(sourcePath, pages, items) {
+    return pages.flatMap((page, i) => items[i]?.length
+        ? [{ page: page.slug, sourcePath, items: items[i], moduleError: null }]
+        : []);
+}
+/** The entry of a page that a file could not make at all, with the errors that stopped it. */
+export function failedPage(page, sourcePath, errors) {
+    return {
+        page,
+        sourcePath,
+        items: [],
+        moduleError: {
+            sourcePath,
+            errors: errors.filter((error) => error.severity === "error"),
+        },
+    };
+}
+/**
+ * The entry of a module that made no page, under the slug its file declares.
+ * None when it declares no slug (a frontmatter error, for example): readers
+ * find an entry by its page's slug.
+ */
+export function failedModule(content, path, errors) {
+    const { slug } = parseFrontmatter(content, path).frontmatter;
+    return typeof slug === "string" && slug.trim()
+        ? [failedPage(slug, path, errors)]
+        : [];
+}
+/**
+ * The "dropped" artifact: page → entry, in path order. A later page with the
+ * same slug wins, as in the platform's page table, except that a module that
+ * made no page never hides a page that exists. Each error is the copy in the
+ * final error list, which carries its category. An item that repeats another
+ * of its page exactly (two excerpts of one missing article, say) is shown once.
+ */
+export function droppedArtifact(entries, errors) {
+    const final = reportedErrors(errors);
+    const artifact = {};
+    for (const entry of entries) {
+        const held = artifact[entry.page];
+        if (held && !held.moduleError && entry.moduleError)
+            continue;
+        const seen = new Set();
+        artifact[entry.page] = {
+            page: entry.page,
+            sourcePath: entry.sourcePath,
+            items: entry.items.flatMap((item) => {
+                const { afterSection, what, error } = item;
+                const key = JSON.stringify([afterSection, what, errorKey(error)]);
+                if (seen.has(key))
+                    return [];
+                seen.add(key);
+                return [{ ...item, error: final(error) }];
+            }),
+            moduleError: entry.moduleError && {
+                sourcePath: entry.moduleError.sourcePath,
+                errors: entry.moduleError.errors.map(final),
+            },
+        };
+    }
+    return artifact;
+}
+/**
+ * The copy of a recorded error in the final error list: the one with the same
+ * dedupe key. The bundler numbers an article's inline element from the start
+ * of the published body; the article's own check, which reports the
+ * element's parse errors, numbers it from the start of the file. So when no
+ * key matches, the reported error with the same file, severity and message,
+ * at or after the recorded line, is the one. With neither, the recorded error
+ * stays as it is (nothing reported it).
+ */
+function reportedErrors(errors) {
+    const byKey = new Map(errors.map((error) => [errorKey(error), error]));
+    const sameText = (error) => [error.file, error.severity, error.message].join("\x1f");
+    const byText = new Map();
+    for (const error of errors) {
+        const list = byText.get(sameText(error)) ?? [];
+        list.push(error);
+        byText.set(sameText(error), list);
+    }
+    for (const list of byText.values()) {
+        list.sort((a, b) => (a.line ?? 0) - (b.line ?? 0));
+    }
+    return (error) => byKey.get(errorKey(error)) ??
+        byText
+            .get(sameText(error))
+            ?.find((candidate) => (candidate.line ?? 0) >= (error.line ?? 0)) ??
+        error;
+}
+//# sourceMappingURL=dropped.js.map

package/dist/dropped.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dropped.js","sourceRoot":"","sources":["../src/dropped.ts"],"names":[],"mappings":"AAAA,yEAAyE;AACzE,6EAA6E;AAC7E,gCAAgC;AAChC,EAAE;AACF,0EAA0E;AAC1E,6BAA6B;AAC7B,2EAA2E;AAC3E,8EAA8E;AAC9E,8EAA8E;AAC9E,0EAA0E;AAC1E,uEAAuE;AACvE,2DAA2D;AAC3D,2DAA2D;AAC3D,6EAA6E;AAC7E,2EAA2E;AAC3E,kEAAkE;AAClE,yEAAyE;AACzE,oEAAoE;AACpE,iCAAiC;AACjC,EAAE;AACF,0EAA0E;AAC1E,6EAA6E;AAC7E,yEAAyE;AACzE,8BAA8B;AAG9B,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC3D,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAqB1E,sEAAsE;AACtE,MAAM,UAAU,WAAW,CACzB,KAAa,EACb,QAA4B,EAC5B,IAAoB;IAEpB,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,IAAI,IAAI,CAAC;IAC1D,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,IAAI,EAAE,CAAC;AAC1D,CAAC;AAWD,MAAM,UAAU,cAAc,CAC5B,IAAiB,EACjB,KAAa,EACb,UAAkB,EAClB,KAAmB;IAEnB,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC;AACtD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CACvB,MAA+B;IAE/B,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,0GAA0G;AAC1G,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,2EAA2E;AAC3E,2EAA2E;AAC3E,MAAM,WAAW,GAAG,MAAM,CAAC,8BAA8B,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAG,MAAM,CAAC,+BAA+B,CAAC,CAAC;AAEzD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAqB,EACrB,IAAoB;IAEpB,MAAM,WAAW,GAAY;QAC3B,IAAI;QACJ,IAAI,EAAE,EAAE;QACR,QAAQ,EAAE,EAAE;QACZ,SAAS,EAAE,IAAI;QACf,gBAAgB,EAAE,EAAE;KACrB,CAAC;IACF,OAAO,MAAM,CAAC,cAAc,CAAC,WAAW,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC1E,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,mHAAmH;AACnH,MAAM,UAAU,qBAAqB,CACnC,OAAU,EACV,KAAgC;IAEhC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACvC,OAAO,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,EAAE;QAC9C,KAAK,EAAE,CAAC,GAAG,iBAAiB,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,CAAC;QAChD,YAAY,EAAE,IAAI;KACnB,CAAC,CAAC;AACL,CAAC;AAED,8GAA8G;AAC9G,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,OAAQ,KAA2C,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;AACtE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,MAA0B,EAC1B,IAAY,EACZ,MAA+B;IAE/B,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,MAAM,IAAI,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACrD,MAAM,UAAU,GACd,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI;QACpC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC;QACxC,CAAC,CAAC,IAAI,CAAC;IACX,MAAM,KAAK,GACT,UAAU,KAAK,IAAI;QACjB,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC;QACtB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5C,OAAO,CAAC,cAAc,CAAC,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,QAA4B;IAIvD,MAAM,IAAI,GAAc,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,IAAoB,EAAE,EAAE,CACrC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IACvD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAI,OAA8C,CAAC,WAAW,CAAC,CAAC;QAC1E,IAAI,IAAI,EAAE,CAAC;YACT,KAAK,CAAC,IAAI,CAAC,CAAC;YACZ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACnB,iBAAiB,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACnC,CAAC;AAaD;;;;;;;;GAQG;AACH,MAAM,UAAU,UAAU,CACxB,UAAkB,EAClB,KAAkD,EAClD,KAAsB,EACtB,QAAoC,EACpC,OAA2C;IAE3C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,MAAM,GAAG,GAAG,CAAC,IAAwB,EAAE,EAAE;QACvC,IAAI,IAAI,KAAK,SAAS;YAAE,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9E,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAC1E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC3E,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC3C,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzB,CAAC;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACpD,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAC5C,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACtD,MAAM,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,KAAK,GAAiB;YAC1B,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,IAAI,EAAE,2BAA2B;YACjC,OAAO,EAAE,8DAA8D;YACvE,UAAU,EACR,mEAAmE;YACrE,QAAQ,EAAE,OAAO;SAClB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CACd,WAAW,CACT,QAAQ,CAAC,MAAM,GAAG,CAAC,EACnB,QAAQ,EACR,cAAc,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CACpE,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,WAAW,CACzB,UAAkB,EAClB,KAAkC,EAClC,KAA+B;IAE/B,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC/B,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM;QACd,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QACvE,CAAC,CAAC,EAAE,CACP,CAAC;AACJ,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,UAAU,CACxB,IAAY,EACZ,UAAkB,EAClB,MAA+B;IAE/B,OAAO;QACL,IAAI;QACJ,UAAU;QACV,KAAK,EAAE,EAAE;QACT,WAAW,EAAE;YACX,UAAU;YACV,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC;SAC7D;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAe,EACf,IAAY,EACZ,MAA+B;IAE/B,MAAM,EAAE,IAAI,EAAE,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,WAAW,CAAC;IAC7D,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE;QAC5C,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC,CAAC,EAAE,CAAC;AACT,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAiC,EAAE,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW;YAAE,SAAS;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG;YACrB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAClC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC;gBAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,YAAY,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAClE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,OAAO,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5C,CAAC,CAAC;YACF,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI;gBAChC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,UAAU;gBACxC,MAAM,EAAE,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;aAC5C;SACF,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,cAAc,CACrB,MAA+B;IAE/B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,CAAC,KAAmB,EAAE,EAAE,CACvC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,CAAC,KAAK,EAAE,EAAE,CACf,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM;aACH,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrB,EAAE,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;QACnE,KAAK,CAAC;AACV,CAAC"}

package/dist/engine/assemble.d.ts

@@ -0,0 +1,44 @@
+import { type ContentError, type PageFinish, type WholeContent } from "../index.js";
+import { type DeferredUrlReport } from "../cli-steps.js";
+import { type FileReferences } from "../reference-graph.js";
+import { type DroppedEntry } from "../dropped.js";
+import type { ContentState } from "./reader.js";
+import { type BuildStore } from "./store.js";
+/**
+ * Lay out the errors, write the artifacts whose JSON changed, and compute
+ * the URL report. The stored hash of each artifact (store.ts) tells a change
+ * without reading the stored JSON back.
+ */
+export declare function assemble(store: BuildStore, units: {
+    source: string;
+    errors: ContentError[];
+    dropped?: DroppedEntry[];
+}[], whole: WholeContent, pages: {
+    source: string;
+    finish: PageFinish;
+}[], state: ContentState, references: ReadonlyMap<string, FileReferences>): {
+    errors: ContentError[];
+    changedArtifacts: string[];
+    urlReport: DeferredUrlReport;
+};
+/**
+ * Write the complete output of `cli.ts --stdin --defer-urls` for the build
+ * layer's content; without `courseGraph`, that of `--skip-course-graph` too.
+ * The pages (141 of the 157 MB on the frozen content) are read from the store
+ * one at a time as they are written, so the job holds one page, not all.
+ */
+export declare function writeFullOutput(path: string, store: BuildStore, whole: WholeContent, errors: ContentError[], texts: ReadonlyMap<string, string>, { courseGraph }: {
+    courseGraph: boolean;
+}): void;
+/** An array that writeJson writes one item at a time, as `items` yields them. */
+export declare class Streamed {
+    readonly items: Iterable<unknown>;
+    constructor(items: Iterable<unknown>);
+}
+/**
+ * Write `JSON.stringify(object, null, 2)` to `path` one top-level value at a
+ * time, and a Streamed value one item at a time. JSON text has no raw newline
+ * inside a string, so a value serialised on its own takes its nested place
+ * once each line after its first is indented to its depth.
+ */
+export declare function writeJson(path: string, object: object): void;

package/dist/engine/assemble.js

@@ -0,0 +1,132 @@
+// What the engine hands out after a job: the error list in today's order with
+// the units that reported each error, the artifacts the platform reads, the
+// URL report, and (for tests, the harness and shadow comparisons) the
+// complete output of `cli.ts --stdin --defer-urls`, byte for byte.
+import { closeSync, openSync, writeFileSync } from "fs";
+import { assembleErrors, processResult, } from "../index.js";
+import { courseReachability, deferredUrlReport, withCourseGraph, } from "../cli-steps.js";
+import { fileToCoursesFrom } from "../reference-graph.js";
+import { droppedArtifact } from "../dropped.js";
+import { NO_ARTIFACT, artifactHash } from "./store.js";
+/** The artifacts the platform reads (protocol.ts, "Artifacts"). */
+const PLATFORM_ARTIFACTS = [
+    "errors",
+    "errorSources",
+    "courses",
+    "surveys",
+    "meetingDocs",
+    "articleSegments",
+    "urlsToValidate",
+    "skillTree",
+    "fileToCourses",
+    "dropped",
+];
+/**
+ * Lay out the errors, write the artifacts whose JSON changed, and compute
+ * the URL report. The stored hash of each artifact (store.ts) tells a change
+ * without reading the stored JSON back.
+ */
+export function assemble(store, units, whole, pages, state, references) {
+    const views = state.views();
+    const { errors, sources } = assembleErrors(units, whole.errors, pages, views.tierMap);
+    const output = processResult([], whole, errors);
+    const artifacts = {
+        errors,
+        errorSources: sources,
+        courses: output.courses,
+        surveys: output.surveys,
+        meetingDocs: output.meetingDocs,
+        articleSegments: output.articleSegments,
+        urlsToValidate: output.urlsToValidate,
+        skillTree: output.skillTree,
+        fileToCourses: courseReachability(() => fileToCoursesFrom(references, views.files)),
+        dropped: droppedArtifact(units.flatMap((unit) => unit.dropped ?? []), errors),
+    };
+    const changedArtifacts = [];
+    for (const name of PLATFORM_ARTIFACTS) {
+        const json = JSON.stringify(artifacts[name]);
+        const hash = json === undefined ? NO_ARTIFACT : artifactHash(json);
+        if (hash === store.getArtifactHash(name))
+            continue;
+        if (json === undefined)
+            store.deleteArtifact(name);
+        else
+            store.putArtifact(name, json);
+        changedArtifacts.push(name);
+    }
+    return {
+        errors,
+        changedArtifacts,
+        urlReport: deferredUrlReport(errors, output.urlsToValidate),
+    };
+}
+/**
+ * Write the complete output of `cli.ts --stdin --defer-urls` for the build
+ * layer's content; without `courseGraph`, that of `--skip-course-graph` too.
+ * The pages (141 of the 157 MB on the frozen content) are read from the store
+ * one at a time as they are written, so the job holds one page, not all.
+ */
+export function writeFullOutput(path, store, whole, errors, texts, { courseGraph }) {
+    const full = {
+        // processResult gives the keys their order.
+        ...processResult([], whole, errors),
+        modules: new Streamed(storedPages(store)),
+        errors: courseGraph ? withCourseGraph(errors, texts) : errors,
+        fileToCourses: JSON.parse(store.getArtifact("fileToCourses") ?? "{}"),
+    };
+    writeJson(path, {
+        ...full,
+        ...deferredUrlReport(full.errors, full.urlsToValidate),
+    });
+}
+/** The finished pages in today's order, each read when it is needed. */
+function* storedPages(store) {
+    for (const { sourcePath, index } of store.listPages())
+        yield JSON.parse(store.getPage(sourcePath, index).json);
+}
+/** An array that writeJson writes one item at a time, as `items` yields them. */
+export class Streamed {
+    items;
+    constructor(items) {
+        this.items = items;
+    }
+}
+/**
+ * Write `JSON.stringify(object, null, 2)` to `path` one top-level value at a
+ * time, and a Streamed value one item at a time. JSON text has no raw newline
+ * inside a string, so a value serialised on its own takes its nested place
+ * once each line after its first is indented to its depth.
+ */
+export function writeJson(path, object) {
+    const nested = (value, indent) => JSON.stringify(value, null, 2)?.replaceAll("\n", `\n${indent}`);
+    const fd = openSync(path, "w");
+    try {
+        const write = (text) => writeFileSync(fd, text);
+        let comma = "";
+        write("{");
+        for (const [key, value] of Object.entries(object)) {
+            const name = `${comma}\n  ${JSON.stringify(key)}: `;
+            if (value instanceof Streamed) {
+                write(`${name}[`);
+                let itemComma = "";
+                for (const item of value.items) {
+                    write(`${itemComma}\n    ${nested(item, "    ") ?? "null"}`);
+                    itemComma = ",";
+                }
+                write(itemComma ? "\n  ]" : "]");
+            }
+            else {
+                const json = nested(value, "  ");
+                if (json === undefined)
+                    continue;
+                write(name + json);
+            }
+            comma = ",";
+        }
+        write(comma ? "\n}" : "}");
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+//# sourceMappingURL=assemble.js.map

package/dist/engine/assemble.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assemble.js","sourceRoot":"","sources":["../../src/engine/assemble.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,4EAA4E;AAC5E,sEAAsE;AACtE,mEAAmE;AAEnE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACxD,OAAO,EACL,cAAc,EACd,aAAa,GAId,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,eAAe,GAEhB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,iBAAiB,EAAuB,MAAM,uBAAuB,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAqB,MAAM,eAAe,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAmB,MAAM,YAAY,CAAC;AAExE,mEAAmE;AACnE,MAAM,kBAAkB,GAAG;IACzB,QAAQ;IACR,cAAc;IACd,SAAS;IACT,SAAS;IACT,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,WAAW;IACX,eAAe;IACf,SAAS;CACD,CAAC;AAEX;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CACtB,KAAiB,EACjB,KAA6E,EAC7E,KAAmB,EACnB,KAA+C,EAC/C,KAAmB,EACnB,UAA+C;IAM/C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5B,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CACxC,KAAK,EACL,KAAK,CAAC,MAAM,EACZ,KAAK,EACL,KAAK,CAAC,OAAO,CACd,CAAC;IACF,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,SAAS,GAAyD;QACtE,MAAM;QACN,YAAY,EAAE,OAAO;QACrB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,aAAa,EAAE,kBAAkB,CAAC,GAAG,EAAE,CACrC,iBAAiB,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,CAC3C;QACD,OAAO,EAAE,eAAe,CACtB,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAC3C,MAAM,CACP;KACF,CAAC;IACF,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,kBAAkB,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnE,IAAI,IAAI,KAAK,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC;YAAE,SAAS;QACnD,IAAI,IAAI,KAAK,SAAS;YAAE,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;;YAC9C,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO;QACL,MAAM;QACN,gBAAgB;QAChB,SAAS,EAAE,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;KAC5D,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,KAAiB,EACjB,KAAmB,EACnB,MAAsB,EACtB,KAAkC,EAClC,EAAE,WAAW,EAA4B;IAEzC,MAAM,IAAI,GAAG;QACX,4CAA4C;QAC5C,GAAG,aAAa,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC;QACnC,OAAO,EAAE,IAAI,QAAQ,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM;QAC7D,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,eAAe,CAAC,IAAI,IAAI,CAAC;KACtE,CAAC;IACF,SAAS,CAAC,IAAI,EAAE;QACd,GAAG,IAAI;QACP,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC;KACvD,CAAC,CAAC;AACL,CAAC;AAED,wEAAwE;AACxE,QAAQ,CAAC,CAAC,WAAW,CAAC,KAAiB;IACrC,KAAK,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,KAAK,CAAC,SAAS,EAAE;QACnD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAE,CAAC,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED,iFAAiF;AACjF,MAAM,OAAO,QAAQ;IACE;IAArB,YAAqB,KAAwB;QAAxB,UAAK,GAAL,KAAK,CAAmB;IAAG,CAAC;CAClD;AAED;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,MAAc;IACpD,MAAM,MAAM,GAAG,CAAC,KAAc,EAAE,MAAc,EAAsB,EAAE,CACpE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,UAAU,CAAC,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC;IAClE,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACxD,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,KAAK,CAAC,GAAG,CAAC,CAAC;QACX,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,GAAG,GAAG,KAAK,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YACpD,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;gBAC9B,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,GAAG,EAAE,CAAC;gBACnB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;oBAC/B,KAAK,CAAC,GAAG,SAAS,SAAS,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC;oBAC7D,SAAS,GAAG,GAAG,CAAC;gBAClB,CAAC;gBACD,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACjC,IAAI,IAAI,KAAK,SAAS;oBAAE,SAAS;gBACjC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;YACrB,CAAC;YACD,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,SAAS,CAAC,EAAE,CAAC,CAAC;IAChB,CAAC;AACH,CAAC"}

package/dist/engine/build.d.ts

@@ -0,0 +1,8 @@
+import type { ContentSource, Job, JobHooks, JobResult } from "./protocol.js";
+import { type BuildStore } from "./store.js";
+/**
+ * Run one job. Throws when the job cannot complete; the worker reports that
+ * as a result with status "error". The store must belong to the running
+ * validator version.
+ */
+export declare function runJob(job: Job, store: BuildStore, source: ContentSource, hooks?: JobHooks): JobResult;