lens-content-processor 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundler/article.d.ts +7 -2
- package/dist/bundler/article.js +19 -8
- package/dist/bundler/article.js.map +1 -1
- package/dist/bundler/video.d.ts +12 -6
- package/dist/bundler/video.js +57 -34
- package/dist/bundler/video.js.map +1 -1
- package/dist/cli.d.ts +11 -2
- package/dist/cli.js +36 -19
- package/dist/cli.js.map +1 -1
- package/dist/content-schema.js +64 -15
- package/dist/content-schema.js.map +1 -1
- package/dist/dedupe-errors.d.ts +14 -0
- package/dist/dedupe-errors.js +33 -0
- package/dist/dedupe-errors.js.map +1 -0
- package/dist/flattener/index.d.ts +22 -28
- package/dist/flattener/index.js +773 -613
- package/dist/flattener/index.js.map +1 -1
- package/dist/flattener/resolve-text-links.d.ts +25 -0
- package/dist/flattener/resolve-text-links.js +248 -0
- package/dist/flattener/resolve-text-links.js.map +1 -0
- package/dist/flattener/test-helpers.d.ts +23 -0
- package/dist/flattener/test-helpers.js +56 -0
- package/dist/flattener/test-helpers.js.map +1 -0
- package/dist/index.d.ts +66 -20
- package/dist/index.js +327 -152
- package/dist/index.js.map +1 -1
- package/dist/parser/article.d.ts +3 -2
- package/dist/parser/article.js +80 -12
- package/dist/parser/article.js.map +1 -1
- package/dist/parser/course.d.ts +1 -1
- package/dist/parser/course.js +88 -28
- package/dist/parser/course.js.map +1 -1
- package/dist/parser/frontmatter.d.ts +1 -1
- package/dist/parser/frontmatter.js +38 -12
- package/dist/parser/frontmatter.js.map +1 -1
- package/dist/parser/learning-outcome.d.ts +12 -2
- package/dist/parser/learning-outcome.js +177 -96
- package/dist/parser/learning-outcome.js.map +1 -1
- package/dist/parser/lens.d.ts +52 -28
- package/dist/parser/lens.js +283 -260
- package/dist/parser/lens.js.map +1 -1
- package/dist/parser/module.d.ts +7 -15
- package/dist/parser/module.js +139 -170
- package/dist/parser/module.js.map +1 -1
- package/dist/parser/sections.d.ts +14 -1
- package/dist/parser/sections.js +94 -81
- package/dist/parser/sections.js.map +1 -1
- package/dist/parser/video-transcript.d.ts +2 -1
- package/dist/parser/video-transcript.js +10 -6
- package/dist/parser/video-transcript.js.map +1 -1
- package/dist/reference-graph.d.ts +8 -0
- package/dist/reference-graph.js +105 -0
- package/dist/reference-graph.js.map +1 -0
- package/dist/validator/card-module-membership.d.ts +2 -0
- package/dist/validator/card-module-membership.js +80 -0
- package/dist/validator/card-module-membership.js.map +1 -0
- package/dist/validator/directives.d.ts +12 -0
- package/dist/validator/directives.js +428 -0
- package/dist/validator/directives.js.map +1 -0
- package/dist/validator/field-values.js +14 -0
- package/dist/validator/field-values.js.map +1 -1
- package/dist/validator/output-integrity.js +5 -5
- package/dist/validator/output-integrity.js.map +1 -1
- package/dist/validator/segment-fields.d.ts +1 -1
- package/dist/validator/segment-fields.js +2 -2
- package/dist/validator/segment-fields.js.map +1 -1
- package/dist/validator/test-segments.d.ts +12 -0
- package/dist/validator/test-segments.js +40 -0
- package/dist/validator/test-segments.js.map +1 -0
- package/dist/validator/url-reachability.d.ts +2 -1
- package/dist/validator/url-reachability.js +42 -20
- package/dist/validator/url-reachability.js.map +1 -1
- package/dist/validator/uuid.js +9 -6
- package/dist/validator/uuid.js.map +1 -1
- package/package.json +1 -1
- package/dist/validator/chat-precedence.d.ts +0 -9
- package/dist/validator/chat-precedence.js +0 -32
- package/dist/validator/chat-precedence.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,95 +1,165 @@
|
|
|
1
|
-
import { flattenModule, flattenLens } from
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
|
|
1
|
+
import { flattenModule, flattenLens } from "./flattener/index.js";
|
|
2
|
+
import { dedupeErrors } from "./dedupe-errors.js";
|
|
3
|
+
import { populateCardModuleSlugs, resolveInlineLensModuleSlugs, } from "./flattener/resolve-text-links.js";
|
|
4
|
+
import { parseModule, hasFieldBeforeSegmentHeaders } from "./parser/module.js";
|
|
5
|
+
import { parseCourse } from "./parser/course.js";
|
|
6
|
+
import { parseLearningOutcome, } from "./parser/learning-outcome.js";
|
|
7
|
+
import { parseLens } from "./parser/lens.js";
|
|
8
|
+
import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, } from "./parser/wikilink.js";
|
|
9
|
+
import { validateUuids } from "./validator/uuid.js";
|
|
10
|
+
import { detectDuplicateSlugs, } from "./validator/duplicates.js";
|
|
11
|
+
import { validateOutputIntegrity } from "./validator/output-integrity.js";
|
|
12
|
+
import { extractArticleExcerpt } from "./bundler/article.js";
|
|
13
|
+
import { extractVideoExcerpt, } from "./bundler/video.js";
|
|
14
|
+
import { parseArticle } from "./parser/article.js";
|
|
15
|
+
import { parseVideoTranscript } from "./parser/video-transcript.js";
|
|
16
|
+
import { validateTimestamps } from "./validator/timestamps.js";
|
|
17
|
+
import { levenshtein } from "./validator/field-typos.js";
|
|
18
|
+
import { buildTierMap, checkTierViolation, } from "./validator/tier.js";
|
|
19
|
+
import { validateCardModuleMembership } from "./validator/card-module-membership.js";
|
|
20
|
+
export { checkTierViolation } from "./validator/tier.js";
|
|
21
|
+
/**
|
|
22
|
+
* Recursively collect UUIDs from inline lens sections (including submodule children).
|
|
23
|
+
*/
|
|
24
|
+
function collectInlineLensUUIDs(sections, path, uuidEntries, errors) {
|
|
25
|
+
for (const section of sections) {
|
|
26
|
+
if (section.type === "lens") {
|
|
27
|
+
if (section.inlineLens) {
|
|
28
|
+
// Inline lens — id already validated in parseInlineLens
|
|
29
|
+
uuidEntries.push({
|
|
30
|
+
uuid: section.fields.id,
|
|
31
|
+
file: path,
|
|
32
|
+
field: "section id",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
else if (!hasFieldBeforeSegmentHeaders(section.body, "source", section.level) &&
|
|
36
|
+
!section.fields.id) {
|
|
37
|
+
// No section-level source:: and no id:: → missing required id
|
|
38
|
+
errors.push({
|
|
39
|
+
file: path,
|
|
40
|
+
line: section.line,
|
|
41
|
+
message: `Inline Lens section '${section.title}' is missing required id:: field`,
|
|
42
|
+
suggestion: "Add an id:: field with a UUID (e.g., id:: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)",
|
|
43
|
+
severity: "error",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
else if (section.fields.id) {
|
|
47
|
+
// Has id:: — collect UUID
|
|
48
|
+
uuidEntries.push({
|
|
49
|
+
uuid: section.fields.id,
|
|
50
|
+
file: path,
|
|
51
|
+
field: "section id",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (section.children) {
|
|
56
|
+
collectInlineLensUUIDs(section.children, path, uuidEntries, errors);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function collectLearningOutcomeTestUUIDs(learningOutcome, path, uuidEntries) {
|
|
61
|
+
if (learningOutcome.test) {
|
|
62
|
+
uuidEntries.push({
|
|
63
|
+
uuid: learningOutcome.test.id,
|
|
64
|
+
file: path,
|
|
65
|
+
field: "test section id",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
for (const group of learningOutcome.submodules ?? []) {
|
|
69
|
+
if (group.test) {
|
|
70
|
+
uuidEntries.push({
|
|
71
|
+
uuid: group.test.id,
|
|
72
|
+
file: path,
|
|
73
|
+
field: "test section id",
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
18
78
|
/**
|
|
19
79
|
* Validate lens excerpts by checking if source files exist and anchors/timestamps are valid.
|
|
80
|
+
* Iterates flat segments, using each segment's source field (set via inheritance in parseLens).
|
|
20
81
|
*/
|
|
21
82
|
function validateLensExcerpts(lens, lensPath, files, tierMap) {
|
|
22
83
|
const errors = [];
|
|
23
|
-
for (const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// Resolve the source wikilink to get the actual file path
|
|
28
|
-
const wikilink = parseWikilink(section.source);
|
|
29
|
-
if (!wikilink || wikilink.error)
|
|
30
|
-
continue;
|
|
31
|
-
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
32
|
-
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
33
|
-
if (!actualPath) {
|
|
34
|
-
// Find similar files to suggest
|
|
35
|
-
const expectedDir = section.type.includes('article') ? 'articles' : 'video_transcripts';
|
|
36
|
-
const similarFiles = findSimilarFiles(resolvedPath, files, expectedDir);
|
|
37
|
-
const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check that the file exists and the path is correct';
|
|
38
|
-
errors.push({
|
|
39
|
-
file: lensPath,
|
|
40
|
-
line: section.line,
|
|
41
|
-
message: `Source file not found: ${resolvedPath}`,
|
|
42
|
-
suggestion,
|
|
43
|
-
severity: 'error',
|
|
44
|
-
});
|
|
45
|
-
continue;
|
|
46
|
-
}
|
|
47
|
-
// Check tier violation (Lens → Article/Video)
|
|
48
|
-
if (tierMap) {
|
|
49
|
-
const childLabel = section.type.includes('article') ? 'article' : 'video transcript';
|
|
50
|
-
const parentTier = tierMap.get(lensPath) ?? 'production';
|
|
51
|
-
const childTier = tierMap.get(actualPath) ?? 'production';
|
|
52
|
-
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, childLabel, section.line);
|
|
53
|
-
if (violation) {
|
|
54
|
-
errors.push(violation);
|
|
84
|
+
for (const segment of lens.segments) {
|
|
85
|
+
if (segment.type === "article" && segment.source) {
|
|
86
|
+
const wikilink = parseWikilink(segment.source);
|
|
87
|
+
if (!wikilink || wikilink.error)
|
|
55
88
|
continue;
|
|
56
|
-
|
|
57
|
-
|
|
89
|
+
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
90
|
+
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
91
|
+
if (!actualPath) {
|
|
92
|
+
const similarFiles = findSimilarFiles(resolvedPath, files, "articles");
|
|
93
|
+
const suggestion = formatSuggestion(similarFiles, lensPath) ??
|
|
94
|
+
"Check that the file exists and the path is correct";
|
|
95
|
+
errors.push({
|
|
96
|
+
file: lensPath,
|
|
97
|
+
message: `Source file not found: ${resolvedPath}`,
|
|
98
|
+
suggestion,
|
|
99
|
+
severity: "error",
|
|
100
|
+
});
|
|
58
101
|
continue;
|
|
59
102
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (result.error) {
|
|
68
|
-
errors.push({ ...result.error, file: lensPath });
|
|
69
|
-
}
|
|
103
|
+
if (tierMap) {
|
|
104
|
+
const parentTier = tierMap.get(lensPath) ?? "production";
|
|
105
|
+
const childTier = tierMap.get(actualPath) ?? "production";
|
|
106
|
+
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, "article");
|
|
107
|
+
if (violation) {
|
|
108
|
+
errors.push(violation);
|
|
109
|
+
continue;
|
|
70
110
|
}
|
|
111
|
+
if (childTier === "ignored")
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
const sourceContent = files.get(actualPath);
|
|
115
|
+
const result = extractArticleExcerpt(sourceContent, segment.fromAnchor, segment.toAnchor, actualPath);
|
|
116
|
+
if (result.error) {
|
|
117
|
+
errors.push({ ...result.error, file: lensPath });
|
|
71
118
|
}
|
|
72
119
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
120
|
+
else if (segment.type === "video" && segment.source) {
|
|
121
|
+
const wikilink = parseWikilink(segment.source);
|
|
122
|
+
if (!wikilink || wikilink.error)
|
|
123
|
+
continue;
|
|
124
|
+
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
125
|
+
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
126
|
+
if (!actualPath) {
|
|
127
|
+
const similarFiles = findSimilarFiles(resolvedPath, files, "video_transcripts");
|
|
128
|
+
const suggestion = formatSuggestion(similarFiles, lensPath) ??
|
|
129
|
+
"Check that the file exists and the path is correct";
|
|
130
|
+
errors.push({
|
|
131
|
+
file: lensPath,
|
|
132
|
+
message: `Source file not found: ${resolvedPath}`,
|
|
133
|
+
suggestion,
|
|
134
|
+
severity: "error",
|
|
135
|
+
});
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (tierMap) {
|
|
139
|
+
const parentTier = tierMap.get(lensPath) ?? "production";
|
|
140
|
+
const childTier = tierMap.get(actualPath) ?? "production";
|
|
141
|
+
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, "video transcript");
|
|
142
|
+
if (violation) {
|
|
143
|
+
errors.push(violation);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (childTier === "ignored")
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const timestampsPath = actualPath.replace(/\.md$/, ".timestamps.json");
|
|
77
150
|
let timestamps;
|
|
78
151
|
if (files.has(timestampsPath)) {
|
|
79
152
|
try {
|
|
80
153
|
timestamps = JSON.parse(files.get(timestampsPath));
|
|
81
154
|
}
|
|
82
155
|
catch {
|
|
83
|
-
|
|
156
|
+
/* fall back */
|
|
84
157
|
}
|
|
85
158
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
errors.push({ ...result.error, file: lensPath });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
159
|
+
const sourceContent = files.get(actualPath);
|
|
160
|
+
const result = extractVideoExcerpt(sourceContent, segment.fromTimeStr, segment.toTimeStr, actualPath, timestamps);
|
|
161
|
+
if (result.error) {
|
|
162
|
+
errors.push({ ...result.error, file: lensPath });
|
|
93
163
|
}
|
|
94
164
|
}
|
|
95
165
|
}
|
|
@@ -105,51 +175,58 @@ export function processContent(files) {
|
|
|
105
175
|
const slugToPath = new Map();
|
|
106
176
|
const filePathToSlug = new Map(); // Reverse: file path → slug (survives duplicate slugs)
|
|
107
177
|
const courseSlugToFile = new Map();
|
|
178
|
+
const parentSlugToChildren = new Map();
|
|
108
179
|
// Pre-scan: build tier map from frontmatter tags
|
|
109
180
|
const tierMap = buildTierMap(files);
|
|
110
181
|
// Identify file types by path
|
|
111
182
|
for (const [path, content] of files.entries()) {
|
|
112
183
|
// Skip ignored files entirely
|
|
113
|
-
if (tierMap.get(path) ===
|
|
184
|
+
if (tierMap.get(path) === "ignored") {
|
|
114
185
|
continue;
|
|
115
186
|
}
|
|
116
|
-
if (path.startsWith(
|
|
187
|
+
if (path.startsWith("modules/")) {
|
|
117
188
|
const result = flattenModule(path, files, new Set(), tierMap);
|
|
118
|
-
|
|
119
|
-
modules.push(
|
|
120
|
-
slugToPath.set(
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
189
|
+
for (const mod of result.modules) {
|
|
190
|
+
modules.push(mod);
|
|
191
|
+
slugToPath.set(mod.slug, path);
|
|
192
|
+
// Track parent→children mapping for course expansion
|
|
193
|
+
if (mod.parentSlug) {
|
|
194
|
+
if (!parentSlugToChildren.has(mod.parentSlug)) {
|
|
195
|
+
parentSlugToChildren.set(mod.parentSlug, []);
|
|
196
|
+
}
|
|
197
|
+
parentSlugToChildren.get(mod.parentSlug).push(mod.slug);
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
filePathToSlug.set(path, mod.slug);
|
|
201
|
+
}
|
|
202
|
+
slugEntries.push({ slug: mod.slug, file: path });
|
|
203
|
+
if (mod.contentId) {
|
|
129
204
|
uuidEntries.push({
|
|
130
|
-
uuid:
|
|
205
|
+
uuid: mod.contentId,
|
|
131
206
|
file: path,
|
|
132
|
-
field:
|
|
207
|
+
field: "contentId",
|
|
133
208
|
});
|
|
134
209
|
}
|
|
135
|
-
|
|
136
|
-
|
|
210
|
+
}
|
|
211
|
+
// If no submodules, set filePathToSlug for the primary module
|
|
212
|
+
if (result.modules.length === 1 && !result.modules[0].parentSlug) {
|
|
213
|
+
filePathToSlug.set(path, result.modules[0].slug);
|
|
214
|
+
}
|
|
215
|
+
else if (result.modules.length > 0 && result.modules[0].parentSlug) {
|
|
216
|
+
// For split modules, map the file path to the parent slug
|
|
217
|
+
filePathToSlug.set(path, result.modules[0].parentSlug);
|
|
218
|
+
}
|
|
219
|
+
// Collect section-level id:: fields from Lens sections (inline lenses).
|
|
220
|
+
// Referenced lenses (with source::) get their id from the lens file itself.
|
|
221
|
+
if (result.modules.length > 0) {
|
|
137
222
|
const rawParse = parseModule(content, path);
|
|
138
223
|
if (rawParse.module) {
|
|
139
|
-
|
|
140
|
-
if (section.type === 'page' && section.fields.id) {
|
|
141
|
-
uuidEntries.push({
|
|
142
|
-
uuid: section.fields.id,
|
|
143
|
-
file: path,
|
|
144
|
-
field: 'section id',
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
}
|
|
224
|
+
collectInlineLensUUIDs(rawParse.module.sections, path, uuidEntries, errors);
|
|
148
225
|
}
|
|
149
226
|
}
|
|
150
227
|
errors.push(...result.errors);
|
|
151
228
|
}
|
|
152
|
-
else if (path.startsWith(
|
|
229
|
+
else if (path.startsWith("courses/")) {
|
|
153
230
|
const result = parseCourse(content, path);
|
|
154
231
|
if (result.course) {
|
|
155
232
|
courses.push(result.course);
|
|
@@ -157,35 +234,38 @@ export function processContent(files) {
|
|
|
157
234
|
}
|
|
158
235
|
errors.push(...result.errors);
|
|
159
236
|
}
|
|
160
|
-
else if (path.startsWith(
|
|
237
|
+
else if (path.startsWith("Learning Outcomes/") ||
|
|
238
|
+
path.includes("/Learning Outcomes/")) {
|
|
161
239
|
// Fully validate Learning Outcome (structure, fields, wikilink syntax)
|
|
162
240
|
const result = parseLearningOutcome(content, path);
|
|
163
241
|
errors.push(...result.errors);
|
|
164
242
|
// Check that referenced lens files exist
|
|
165
243
|
if (result.learningOutcome) {
|
|
244
|
+
collectLearningOutcomeTestUUIDs(result.learningOutcome, path, uuidEntries);
|
|
166
245
|
for (const lensRef of result.learningOutcome.lenses) {
|
|
167
246
|
const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
|
|
168
247
|
if (!lensPath) {
|
|
169
248
|
// Find similar files to suggest
|
|
170
|
-
const similarFiles = findSimilarFiles(lensRef.resolvedPath, files,
|
|
171
|
-
const suggestion = formatSuggestion(similarFiles, path) ??
|
|
249
|
+
const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
|
|
250
|
+
const suggestion = formatSuggestion(similarFiles, path) ??
|
|
251
|
+
"Check the file path in the wiki-link";
|
|
172
252
|
errors.push({
|
|
173
253
|
file: path,
|
|
174
254
|
message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
|
|
175
255
|
suggestion,
|
|
176
|
-
severity:
|
|
256
|
+
severity: "error",
|
|
177
257
|
});
|
|
178
258
|
continue;
|
|
179
259
|
}
|
|
180
260
|
// Check tier violation (LO → Lens)
|
|
181
|
-
const parentTier = tierMap.get(path) ??
|
|
182
|
-
const childTier = tierMap.get(lensPath) ??
|
|
183
|
-
const violation = checkTierViolation(path, parentTier, lensPath, childTier,
|
|
261
|
+
const parentTier = tierMap.get(path) ?? "production";
|
|
262
|
+
const childTier = tierMap.get(lensPath) ?? "production";
|
|
263
|
+
const violation = checkTierViolation(path, parentTier, lensPath, childTier, "lens");
|
|
184
264
|
if (violation) {
|
|
185
265
|
errors.push(violation);
|
|
186
266
|
continue;
|
|
187
267
|
}
|
|
188
|
-
if (childTier ===
|
|
268
|
+
if (childTier === "ignored") {
|
|
189
269
|
continue;
|
|
190
270
|
}
|
|
191
271
|
}
|
|
@@ -195,25 +275,20 @@ export function processContent(files) {
|
|
|
195
275
|
uuidEntries.push({
|
|
196
276
|
uuid: result.learningOutcome.id,
|
|
197
277
|
file: path,
|
|
198
|
-
field:
|
|
278
|
+
field: "id",
|
|
199
279
|
});
|
|
200
280
|
}
|
|
201
281
|
}
|
|
202
|
-
else if (path.startsWith(
|
|
282
|
+
else if (path.startsWith("Lenses/") || path.includes("/Lenses/")) {
|
|
203
283
|
// Fully validate Lens (structure, segments, fields)
|
|
204
284
|
const result = parseLens(content, path);
|
|
205
285
|
errors.push(...result.errors);
|
|
206
|
-
// Validate excerpts (source files exist, anchors/timestamps valid)
|
|
207
|
-
if (result.lens) {
|
|
208
|
-
const excerptErrors = validateLensExcerpts(result.lens, path, files, tierMap);
|
|
209
|
-
errors.push(...excerptErrors);
|
|
210
|
-
}
|
|
211
286
|
// Collect id for UUID validation
|
|
212
287
|
if (result.lens?.id) {
|
|
213
288
|
uuidEntries.push({
|
|
214
289
|
uuid: result.lens.id,
|
|
215
290
|
file: path,
|
|
216
|
-
field:
|
|
291
|
+
field: "id",
|
|
217
292
|
});
|
|
218
293
|
}
|
|
219
294
|
// Flatten lens as standalone module (pass pre-parsed lens to avoid re-parsing)
|
|
@@ -227,32 +302,65 @@ export function processContent(files) {
|
|
|
227
302
|
errors.push(...lensModuleResult.errors);
|
|
228
303
|
}
|
|
229
304
|
}
|
|
230
|
-
else if (path.endsWith(
|
|
305
|
+
else if (path.endsWith(".timestamps.json")) {
|
|
231
306
|
const tsErrors = validateTimestamps(content, path);
|
|
232
307
|
errors.push(...tsErrors);
|
|
233
308
|
}
|
|
234
|
-
else if (path.startsWith(
|
|
309
|
+
else if (path.startsWith("articles/") || path.includes("/articles/")) {
|
|
235
310
|
const result = parseArticle(content, path);
|
|
236
311
|
errors.push(...result.errors);
|
|
237
312
|
if (result.article) {
|
|
238
|
-
|
|
313
|
+
// Only carry the flag when set true, so URL entries (and thus the
|
|
314
|
+
// serialized output / Python parity) are unchanged for normal articles.
|
|
315
|
+
const overrideFlag = result.article.allowUnreachableUrl
|
|
316
|
+
? { allowUnreachable: true }
|
|
317
|
+
: {};
|
|
318
|
+
urlsToValidate.push({
|
|
319
|
+
url: result.article.sourceUrl,
|
|
320
|
+
file: path,
|
|
321
|
+
line: 2,
|
|
322
|
+
label: "source_url",
|
|
323
|
+
...overrideFlag,
|
|
324
|
+
});
|
|
239
325
|
for (const img of result.article.imageUrls) {
|
|
240
|
-
urlsToValidate.push({
|
|
326
|
+
urlsToValidate.push({
|
|
327
|
+
url: img.url,
|
|
328
|
+
file: path,
|
|
329
|
+
line: img.line,
|
|
330
|
+
label: "Image URL",
|
|
331
|
+
...overrideFlag,
|
|
332
|
+
});
|
|
241
333
|
}
|
|
242
334
|
}
|
|
243
335
|
}
|
|
244
|
-
else if (path.startsWith(
|
|
336
|
+
else if (path.startsWith("video_transcripts/") ||
|
|
337
|
+
path.includes("/video_transcripts/")) {
|
|
245
338
|
const result = parseVideoTranscript(content, path);
|
|
246
339
|
errors.push(...result.errors);
|
|
247
340
|
if (result.transcript) {
|
|
248
|
-
urlsToValidate.push({
|
|
341
|
+
urlsToValidate.push({
|
|
342
|
+
url: result.transcript.url,
|
|
343
|
+
file: path,
|
|
344
|
+
line: 2,
|
|
345
|
+
label: "url",
|
|
346
|
+
...(result.transcript.allowUnreachableUrl
|
|
347
|
+
? { allowUnreachable: true }
|
|
348
|
+
: {}),
|
|
349
|
+
});
|
|
249
350
|
}
|
|
250
351
|
}
|
|
251
352
|
else {
|
|
252
353
|
// File didn't match any known directory pattern — check for near-misses via Levenshtein distance
|
|
253
|
-
const dir = path.split(
|
|
254
|
-
const VALID_DIRS = [
|
|
255
|
-
|
|
354
|
+
const dir = path.split("/")[0];
|
|
355
|
+
const VALID_DIRS = [
|
|
356
|
+
"modules",
|
|
357
|
+
"courses",
|
|
358
|
+
"articles",
|
|
359
|
+
"Lenses",
|
|
360
|
+
"video_transcripts",
|
|
361
|
+
"Learning Outcomes",
|
|
362
|
+
];
|
|
363
|
+
let closest = "";
|
|
256
364
|
let minDist = Infinity;
|
|
257
365
|
for (const valid of VALID_DIRS) {
|
|
258
366
|
const dist = levenshtein(dir.toLowerCase(), valid.toLowerCase());
|
|
@@ -268,18 +376,38 @@ export function processContent(files) {
|
|
|
268
376
|
file: path,
|
|
269
377
|
message: `File in directory '${dir}/' not recognized as content`,
|
|
270
378
|
suggestion: `Did you mean '${closest}/'?`,
|
|
271
|
-
severity:
|
|
379
|
+
severity: "warning",
|
|
272
380
|
});
|
|
273
381
|
}
|
|
274
382
|
}
|
|
275
383
|
}
|
|
384
|
+
// Build contentId → moduleSlug mapping and populate moduleSlug in cross-module card links.
|
|
385
|
+
// Prefer real modules over standalone lenses (lens/ prefix) — a lens that appears in
|
|
386
|
+
// both a module and as a standalone should map to the module.
|
|
387
|
+
const contentIdToModuleSlug = new Map();
|
|
388
|
+
for (const mod of modules) {
|
|
389
|
+
if (mod.slug.startsWith("lens/"))
|
|
390
|
+
continue; // Skip standalone lenses
|
|
391
|
+
for (const section of mod.sections) {
|
|
392
|
+
if (section.contentId) {
|
|
393
|
+
contentIdToModuleSlug.set(section.contentId, mod.slug);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
for (const mod of modules) {
|
|
398
|
+
populateCardModuleSlugs(mod.sections, contentIdToModuleSlug);
|
|
399
|
+
resolveInlineLensModuleSlugs(mod.sections, contentIdToModuleSlug);
|
|
400
|
+
}
|
|
401
|
+
// Card-to-module membership check. Runs after card link resolution above so
|
|
402
|
+
// every card has its data-lens-card JSON populated, including allowExternal.
|
|
403
|
+
errors.push(...validateCardModuleMembership(modules));
|
|
276
404
|
// Resolve course module paths to frontmatter slugs.
|
|
277
405
|
// Use filePathToSlug (built during module parsing) instead of inverting slugToPath,
|
|
278
406
|
// because slugToPath loses entries when duplicate slugs exist.
|
|
279
407
|
for (const course of courses) {
|
|
280
|
-
const courseFile = courseSlugToFile.get(course.slug) ??
|
|
408
|
+
const courseFile = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
281
409
|
for (const item of course.progression) {
|
|
282
|
-
if (item.type ===
|
|
410
|
+
if (item.type === "module" && item.path) {
|
|
283
411
|
// Resolve wikilink path relative to the course file
|
|
284
412
|
const resolved = resolveWikilinkPath(item.path, courseFile);
|
|
285
413
|
const actualFile = findFileWithExtension(resolved, files);
|
|
@@ -288,10 +416,10 @@ export function processContent(files) {
|
|
|
288
416
|
}
|
|
289
417
|
else {
|
|
290
418
|
// Try matching just the filename stem against module file stems
|
|
291
|
-
const stem = item.path.split(
|
|
419
|
+
const stem = item.path.split("/").pop() ?? item.path;
|
|
292
420
|
let matched = false;
|
|
293
421
|
for (const [filePath, slug] of filePathToSlug.entries()) {
|
|
294
|
-
const fileStem = filePath.replace(/\.md$/,
|
|
422
|
+
const fileStem = filePath.replace(/\.md$/, "").split("/").pop() ?? "";
|
|
295
423
|
if (fileStem === stem) {
|
|
296
424
|
item.slug = slug;
|
|
297
425
|
matched = true;
|
|
@@ -302,8 +430,8 @@ export function processContent(files) {
|
|
|
302
430
|
errors.push({
|
|
303
431
|
file: courseFile,
|
|
304
432
|
message: `Module reference could not be resolved: "${item.path}"`,
|
|
305
|
-
suggestion:
|
|
306
|
-
severity:
|
|
433
|
+
suggestion: "Check that the wikilink path points to an existing module file",
|
|
434
|
+
severity: "error",
|
|
307
435
|
});
|
|
308
436
|
}
|
|
309
437
|
}
|
|
@@ -312,7 +440,23 @@ export function processContent(files) {
|
|
|
312
440
|
}
|
|
313
441
|
}
|
|
314
442
|
// Remove unresolved module items (no slug after resolution)
|
|
315
|
-
course.progression = course.progression.filter(item => item.type !==
|
|
443
|
+
course.progression = course.progression.filter((item) => item.type !== "module" || item.slug !== undefined);
|
|
444
|
+
// Expand split modules: replace parent slug with child submodule slugs
|
|
445
|
+
const expanded = [];
|
|
446
|
+
for (const item of course.progression) {
|
|
447
|
+
if (item.type === "module" &&
|
|
448
|
+
item.slug &&
|
|
449
|
+
parentSlugToChildren.has(item.slug)) {
|
|
450
|
+
const children = parentSlugToChildren.get(item.slug);
|
|
451
|
+
for (const childSlug of children) {
|
|
452
|
+
expanded.push({ ...item, slug: childSlug });
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
else {
|
|
456
|
+
expanded.push(item);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
course.progression = expanded;
|
|
316
460
|
}
|
|
317
461
|
// Check tier violations: Course → Module
|
|
318
462
|
for (const course of courses) {
|
|
@@ -320,21 +464,48 @@ export function processContent(files) {
|
|
|
320
464
|
if (!coursePath)
|
|
321
465
|
continue;
|
|
322
466
|
for (const item of course.progression) {
|
|
323
|
-
if (item.type !==
|
|
467
|
+
if (item.type !== "module" || !item.slug)
|
|
324
468
|
continue;
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
const
|
|
330
|
-
const childTier = tierMap.get(modulePath) ?? 'production';
|
|
331
|
-
const violation = checkTierViolation(coursePath, parentTier, modulePath, childTier, 'module');
|
|
469
|
+
const modulePath = slugToPath.get(item.slug);
|
|
470
|
+
if (modulePath && tierMap.has(modulePath)) {
|
|
471
|
+
const parentTier = tierMap.get(coursePath) ?? "production";
|
|
472
|
+
const childTier = tierMap.get(modulePath) ?? "production";
|
|
473
|
+
const violation = checkTierViolation(coursePath, parentTier, modulePath, childTier, "module");
|
|
332
474
|
if (violation) {
|
|
333
475
|
errors.push(violation);
|
|
334
476
|
}
|
|
335
477
|
}
|
|
336
478
|
}
|
|
337
479
|
}
|
|
480
|
+
// Validate course slug alias collisions
|
|
481
|
+
{
|
|
482
|
+
// Map of all course slugs (primary + aliases) -> source file
|
|
483
|
+
const allCourseSlugs = new Map(); // slug -> file
|
|
484
|
+
// Register primary slugs first
|
|
485
|
+
for (const course of courses) {
|
|
486
|
+
const file = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
487
|
+
allCourseSlugs.set(course.slug, file);
|
|
488
|
+
}
|
|
489
|
+
// Check each alias against all known slugs
|
|
490
|
+
for (const course of courses) {
|
|
491
|
+
const file = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
492
|
+
const tier = tierMap.get(file) ?? "production";
|
|
493
|
+
for (const alias of course.slugAliases ?? []) {
|
|
494
|
+
const existing = allCourseSlugs.get(alias);
|
|
495
|
+
if (existing) {
|
|
496
|
+
errors.push({
|
|
497
|
+
file,
|
|
498
|
+
message: `Course slug alias '${alias}' collides with ${existing === file ? "its own primary slug" : `slug in ${existing}`}`,
|
|
499
|
+
suggestion: `Choose a different alias or remove the conflicting slug`,
|
|
500
|
+
severity: tier === "wip" ? "warning" : "error",
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
else {
|
|
504
|
+
allCourseSlugs.set(alias, file);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
338
509
|
// Validate all collected UUIDs
|
|
339
510
|
const uuidValidation = validateUuids(uuidEntries);
|
|
340
511
|
errors.push(...uuidValidation.errors);
|
|
@@ -342,29 +513,30 @@ export function processContent(files) {
|
|
|
342
513
|
const duplicateSlugErrors = detectDuplicateSlugs(slugEntries);
|
|
343
514
|
errors.push(...duplicateSlugErrors);
|
|
344
515
|
// Validate video transcript / timestamps.json pairing
|
|
345
|
-
const transcriptPaths = [...files.keys()].filter(p => (p.startsWith(
|
|
346
|
-
p.
|
|
347
|
-
|
|
348
|
-
|
|
516
|
+
const transcriptPaths = [...files.keys()].filter((p) => (p.startsWith("video_transcripts/") ||
|
|
517
|
+
p.includes("/video_transcripts/")) &&
|
|
518
|
+
p.endsWith(".md") &&
|
|
519
|
+
tierMap.get(p) !== "ignored");
|
|
520
|
+
const timestampPaths = new Set([...files.keys()].filter((p) => p.endsWith(".timestamps.json")));
|
|
349
521
|
for (const mdPath of transcriptPaths) {
|
|
350
|
-
const expectedTs = mdPath.replace(/\.md$/,
|
|
522
|
+
const expectedTs = mdPath.replace(/\.md$/, ".timestamps.json");
|
|
351
523
|
if (!timestampPaths.has(expectedTs)) {
|
|
352
524
|
errors.push({
|
|
353
525
|
file: mdPath,
|
|
354
526
|
message: `Missing timestamps.json: expected ${expectedTs}`,
|
|
355
|
-
severity:
|
|
527
|
+
severity: "error",
|
|
356
528
|
});
|
|
357
529
|
}
|
|
358
530
|
}
|
|
359
531
|
for (const tsPath of timestampPaths) {
|
|
360
|
-
const expectedMd = tsPath.replace(/\.timestamps\.json$/,
|
|
361
|
-
if (tierMap.get(expectedMd) ===
|
|
532
|
+
const expectedMd = tsPath.replace(/\.timestamps\.json$/, ".md");
|
|
533
|
+
if (tierMap.get(expectedMd) === "ignored")
|
|
362
534
|
continue;
|
|
363
535
|
if (!files.has(expectedMd)) {
|
|
364
536
|
errors.push({
|
|
365
537
|
file: tsPath,
|
|
366
538
|
message: `Orphaned timestamps file: no matching .md transcript found`,
|
|
367
|
-
severity:
|
|
539
|
+
severity: "warning",
|
|
368
540
|
});
|
|
369
541
|
}
|
|
370
542
|
}
|
|
@@ -375,9 +547,12 @@ export function processContent(files) {
|
|
|
375
547
|
for (const error of errors) {
|
|
376
548
|
if (!error.category) {
|
|
377
549
|
const tier = tierMap.get(error.file);
|
|
378
|
-
error.category = tier ===
|
|
550
|
+
error.category = tier === "wip" ? "wip" : "production";
|
|
379
551
|
}
|
|
380
552
|
}
|
|
381
|
-
|
|
553
|
+
// The same file is validated once per pass (standalone + once per module
|
|
554
|
+
// that flattens it), so identical errors are emitted multiple times. Collapse
|
|
555
|
+
// them so each problem is reported once.
|
|
556
|
+
return { modules, courses, errors: dedupeErrors(errors), urlsToValidate };
|
|
382
557
|
}
|
|
383
558
|
//# sourceMappingURL=index.js.map
|