lens-content-processor 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundler/video.d.ts +12 -6
- package/dist/bundler/video.js +57 -34
- package/dist/bundler/video.js.map +1 -1
- package/dist/cli.d.ts +11 -2
- package/dist/cli.js +36 -19
- package/dist/cli.js.map +1 -1
- package/dist/content-schema.js +71 -16
- package/dist/content-schema.js.map +1 -1
- package/dist/dedupe-errors.d.ts +14 -0
- package/dist/dedupe-errors.js +33 -0
- package/dist/dedupe-errors.js.map +1 -0
- package/dist/flattener/index.d.ts +3 -28
- package/dist/flattener/index.js +507 -745
- package/dist/flattener/index.js.map +1 -1
- package/dist/flattener/resolve-text-links.d.ts +25 -0
- package/dist/flattener/resolve-text-links.js +248 -0
- package/dist/flattener/resolve-text-links.js.map +1 -0
- package/dist/flattener/test-helpers.d.ts +1 -1
- package/dist/flattener/test-helpers.js +1 -3
- package/dist/flattener/test-helpers.js.map +1 -1
- package/dist/index.d.ts +53 -21
- package/dist/index.js +285 -140
- package/dist/index.js.map +1 -1
- package/dist/parser/article.d.ts +2 -1
- package/dist/parser/article.js +75 -14
- package/dist/parser/article.js.map +1 -1
- package/dist/parser/course.d.ts +1 -1
- package/dist/parser/course.js +79 -22
- package/dist/parser/course.js.map +1 -1
- package/dist/parser/frontmatter.d.ts +1 -1
- package/dist/parser/frontmatter.js +38 -12
- package/dist/parser/frontmatter.js.map +1 -1
- package/dist/parser/learning-outcome.d.ts +5 -2
- package/dist/parser/learning-outcome.js +75 -37
- package/dist/parser/learning-outcome.js.map +1 -1
- package/dist/parser/lens.d.ts +40 -41
- package/dist/parser/lens.js +365 -349
- package/dist/parser/lens.js.map +1 -1
- package/dist/parser/module.d.ts +7 -15
- package/dist/parser/module.js +135 -171
- package/dist/parser/module.js.map +1 -1
- package/dist/parser/sections.d.ts +12 -1
- package/dist/parser/sections.js +74 -95
- package/dist/parser/sections.js.map +1 -1
- package/dist/parser/video-transcript.d.ts +2 -1
- package/dist/parser/video-transcript.js +8 -7
- package/dist/parser/video-transcript.js.map +1 -1
- package/dist/reference-graph.d.ts +8 -0
- package/dist/reference-graph.js +105 -0
- package/dist/reference-graph.js.map +1 -0
- package/dist/validator/card-module-membership.d.ts +2 -0
- package/dist/validator/card-module-membership.js +80 -0
- package/dist/validator/card-module-membership.js.map +1 -0
- package/dist/validator/output-integrity.js +5 -5
- package/dist/validator/output-integrity.js.map +1 -1
- package/dist/validator/segment-fields.d.ts +1 -1
- package/dist/validator/segment-fields.js +2 -2
- package/dist/validator/segment-fields.js.map +1 -1
- package/dist/validator/test-segments.d.ts +12 -0
- package/dist/validator/test-segments.js +40 -0
- package/dist/validator/test-segments.js.map +1 -0
- package/dist/validator/url-reachability.d.ts +2 -1
- package/dist/validator/url-reachability.js +40 -20
- package/dist/validator/url-reachability.js.map +1 -1
- package/package.json +1 -1
- package/dist/validator/chat-precedence.d.ts +0 -9
- package/dist/validator/chat-precedence.js +0 -32
- package/dist/validator/chat-precedence.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,95 +1,165 @@
|
|
|
1
|
-
import { flattenModule, flattenLens } from
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
|
|
1
|
+
import { flattenModule, flattenLens } from "./flattener/index.js";
|
|
2
|
+
import { dedupeErrors } from "./dedupe-errors.js";
|
|
3
|
+
import { populateCardModuleSlugs, resolveInlineLensModuleSlugs, } from "./flattener/resolve-text-links.js";
|
|
4
|
+
import { parseModule, hasFieldBeforeSegmentHeaders } from "./parser/module.js";
|
|
5
|
+
import { parseCourse } from "./parser/course.js";
|
|
6
|
+
import { parseLearningOutcome, } from "./parser/learning-outcome.js";
|
|
7
|
+
import { parseLens } from "./parser/lens.js";
|
|
8
|
+
import { parseWikilink, resolveWikilinkPath, findFileWithExtension, findSimilarFiles, formatSuggestion, } from "./parser/wikilink.js";
|
|
9
|
+
import { validateUuids } from "./validator/uuid.js";
|
|
10
|
+
import { detectDuplicateSlugs, } from "./validator/duplicates.js";
|
|
11
|
+
import { validateOutputIntegrity } from "./validator/output-integrity.js";
|
|
12
|
+
import { extractArticleExcerpt } from "./bundler/article.js";
|
|
13
|
+
import { extractVideoExcerpt, } from "./bundler/video.js";
|
|
14
|
+
import { parseArticle } from "./parser/article.js";
|
|
15
|
+
import { parseVideoTranscript } from "./parser/video-transcript.js";
|
|
16
|
+
import { validateTimestamps } from "./validator/timestamps.js";
|
|
17
|
+
import { levenshtein } from "./validator/field-typos.js";
|
|
18
|
+
import { buildTierMap, checkTierViolation, } from "./validator/tier.js";
|
|
19
|
+
import { validateCardModuleMembership } from "./validator/card-module-membership.js";
|
|
20
|
+
export { checkTierViolation } from "./validator/tier.js";
|
|
21
|
+
/**
|
|
22
|
+
* Recursively collect UUIDs from inline lens sections (including submodule children).
|
|
23
|
+
*/
|
|
24
|
+
function collectInlineLensUUIDs(sections, path, uuidEntries, errors) {
|
|
25
|
+
for (const section of sections) {
|
|
26
|
+
if (section.type === "lens") {
|
|
27
|
+
if (section.inlineLens) {
|
|
28
|
+
// Inline lens — id already validated in parseInlineLens
|
|
29
|
+
uuidEntries.push({
|
|
30
|
+
uuid: section.fields.id,
|
|
31
|
+
file: path,
|
|
32
|
+
field: "section id",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
else if (!hasFieldBeforeSegmentHeaders(section.body, "source", section.level) &&
|
|
36
|
+
!section.fields.id) {
|
|
37
|
+
// No section-level source:: and no id:: → missing required id
|
|
38
|
+
errors.push({
|
|
39
|
+
file: path,
|
|
40
|
+
line: section.line,
|
|
41
|
+
message: `Inline Lens section '${section.title}' is missing required id:: field`,
|
|
42
|
+
suggestion: "Add an id:: field with a UUID (e.g., id:: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)",
|
|
43
|
+
severity: "error",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
else if (section.fields.id) {
|
|
47
|
+
// Has id:: — collect UUID
|
|
48
|
+
uuidEntries.push({
|
|
49
|
+
uuid: section.fields.id,
|
|
50
|
+
file: path,
|
|
51
|
+
field: "section id",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (section.children) {
|
|
56
|
+
collectInlineLensUUIDs(section.children, path, uuidEntries, errors);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
function collectLearningOutcomeTestUUIDs(learningOutcome, path, uuidEntries) {
|
|
61
|
+
if (learningOutcome.test) {
|
|
62
|
+
uuidEntries.push({
|
|
63
|
+
uuid: learningOutcome.test.id,
|
|
64
|
+
file: path,
|
|
65
|
+
field: "test section id",
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
for (const group of learningOutcome.submodules ?? []) {
|
|
69
|
+
if (group.test) {
|
|
70
|
+
uuidEntries.push({
|
|
71
|
+
uuid: group.test.id,
|
|
72
|
+
file: path,
|
|
73
|
+
field: "test section id",
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
18
78
|
/**
|
|
19
79
|
* Validate lens excerpts by checking if source files exist and anchors/timestamps are valid.
|
|
80
|
+
* Iterates flat segments, using each segment's source field (set via inheritance in parseLens).
|
|
20
81
|
*/
|
|
21
82
|
function validateLensExcerpts(lens, lensPath, files, tierMap) {
|
|
22
83
|
const errors = [];
|
|
23
|
-
for (const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// Resolve the source wikilink to get the actual file path
|
|
28
|
-
const wikilink = parseWikilink(section.source);
|
|
29
|
-
if (!wikilink || wikilink.error)
|
|
30
|
-
continue;
|
|
31
|
-
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
32
|
-
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
33
|
-
if (!actualPath) {
|
|
34
|
-
// Find similar files to suggest
|
|
35
|
-
const expectedDir = section.type.includes('article') ? 'articles' : 'video_transcripts';
|
|
36
|
-
const similarFiles = findSimilarFiles(resolvedPath, files, expectedDir);
|
|
37
|
-
const suggestion = formatSuggestion(similarFiles, lensPath) ?? 'Check that the file exists and the path is correct';
|
|
38
|
-
errors.push({
|
|
39
|
-
file: lensPath,
|
|
40
|
-
line: section.line,
|
|
41
|
-
message: `Source file not found: ${resolvedPath}`,
|
|
42
|
-
suggestion,
|
|
43
|
-
severity: 'error',
|
|
44
|
-
});
|
|
45
|
-
continue;
|
|
46
|
-
}
|
|
47
|
-
// Check tier violation (Lens → Article/Video)
|
|
48
|
-
if (tierMap) {
|
|
49
|
-
const childLabel = section.type.includes('article') ? 'article' : 'video transcript';
|
|
50
|
-
const parentTier = tierMap.get(lensPath) ?? 'production';
|
|
51
|
-
const childTier = tierMap.get(actualPath) ?? 'production';
|
|
52
|
-
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, childLabel, section.line);
|
|
53
|
-
if (violation) {
|
|
54
|
-
errors.push(violation);
|
|
84
|
+
for (const segment of lens.segments) {
|
|
85
|
+
if (segment.type === "article" && segment.source) {
|
|
86
|
+
const wikilink = parseWikilink(segment.source);
|
|
87
|
+
if (!wikilink || wikilink.error)
|
|
55
88
|
continue;
|
|
56
|
-
|
|
57
|
-
|
|
89
|
+
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
90
|
+
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
91
|
+
if (!actualPath) {
|
|
92
|
+
const similarFiles = findSimilarFiles(resolvedPath, files, "articles");
|
|
93
|
+
const suggestion = formatSuggestion(similarFiles, lensPath) ??
|
|
94
|
+
"Check that the file exists and the path is correct";
|
|
95
|
+
errors.push({
|
|
96
|
+
file: lensPath,
|
|
97
|
+
message: `Source file not found: ${resolvedPath}`,
|
|
98
|
+
suggestion,
|
|
99
|
+
severity: "error",
|
|
100
|
+
});
|
|
58
101
|
continue;
|
|
59
102
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (result.error) {
|
|
68
|
-
errors.push({ ...result.error, file: lensPath });
|
|
69
|
-
}
|
|
103
|
+
if (tierMap) {
|
|
104
|
+
const parentTier = tierMap.get(lensPath) ?? "production";
|
|
105
|
+
const childTier = tierMap.get(actualPath) ?? "production";
|
|
106
|
+
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, "article");
|
|
107
|
+
if (violation) {
|
|
108
|
+
errors.push(violation);
|
|
109
|
+
continue;
|
|
70
110
|
}
|
|
111
|
+
if (childTier === "ignored")
|
|
112
|
+
continue;
|
|
113
|
+
}
|
|
114
|
+
const sourceContent = files.get(actualPath);
|
|
115
|
+
const result = extractArticleExcerpt(sourceContent, segment.fromAnchor, segment.toAnchor, actualPath);
|
|
116
|
+
if (result.error) {
|
|
117
|
+
errors.push({ ...result.error, file: lensPath });
|
|
71
118
|
}
|
|
72
119
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
120
|
+
else if (segment.type === "video" && segment.source) {
|
|
121
|
+
const wikilink = parseWikilink(segment.source);
|
|
122
|
+
if (!wikilink || wikilink.error)
|
|
123
|
+
continue;
|
|
124
|
+
const resolvedPath = resolveWikilinkPath(wikilink.path, lensPath);
|
|
125
|
+
const actualPath = findFileWithExtension(resolvedPath, files);
|
|
126
|
+
if (!actualPath) {
|
|
127
|
+
const similarFiles = findSimilarFiles(resolvedPath, files, "video_transcripts");
|
|
128
|
+
const suggestion = formatSuggestion(similarFiles, lensPath) ??
|
|
129
|
+
"Check that the file exists and the path is correct";
|
|
130
|
+
errors.push({
|
|
131
|
+
file: lensPath,
|
|
132
|
+
message: `Source file not found: ${resolvedPath}`,
|
|
133
|
+
suggestion,
|
|
134
|
+
severity: "error",
|
|
135
|
+
});
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (tierMap) {
|
|
139
|
+
const parentTier = tierMap.get(lensPath) ?? "production";
|
|
140
|
+
const childTier = tierMap.get(actualPath) ?? "production";
|
|
141
|
+
const violation = checkTierViolation(lensPath, parentTier, actualPath, childTier, "video transcript");
|
|
142
|
+
if (violation) {
|
|
143
|
+
errors.push(violation);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (childTier === "ignored")
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const timestampsPath = actualPath.replace(/\.md$/, ".timestamps.json");
|
|
77
150
|
let timestamps;
|
|
78
151
|
if (files.has(timestampsPath)) {
|
|
79
152
|
try {
|
|
80
153
|
timestamps = JSON.parse(files.get(timestampsPath));
|
|
81
154
|
}
|
|
82
155
|
catch {
|
|
83
|
-
|
|
156
|
+
/* fall back */
|
|
84
157
|
}
|
|
85
158
|
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
errors.push({ ...result.error, file: lensPath });
|
|
91
|
-
}
|
|
92
|
-
}
|
|
159
|
+
const sourceContent = files.get(actualPath);
|
|
160
|
+
const result = extractVideoExcerpt(sourceContent, segment.fromTimeStr, segment.toTimeStr, actualPath, timestamps);
|
|
161
|
+
if (result.error) {
|
|
162
|
+
errors.push({ ...result.error, file: lensPath });
|
|
93
163
|
}
|
|
94
164
|
}
|
|
95
165
|
}
|
|
@@ -111,10 +181,10 @@ export function processContent(files) {
|
|
|
111
181
|
// Identify file types by path
|
|
112
182
|
for (const [path, content] of files.entries()) {
|
|
113
183
|
// Skip ignored files entirely
|
|
114
|
-
if (tierMap.get(path) ===
|
|
184
|
+
if (tierMap.get(path) === "ignored") {
|
|
115
185
|
continue;
|
|
116
186
|
}
|
|
117
|
-
if (path.startsWith(
|
|
187
|
+
if (path.startsWith("modules/")) {
|
|
118
188
|
const result = flattenModule(path, files, new Set(), tierMap);
|
|
119
189
|
for (const mod of result.modules) {
|
|
120
190
|
modules.push(mod);
|
|
@@ -134,7 +204,7 @@ export function processContent(files) {
|
|
|
134
204
|
uuidEntries.push({
|
|
135
205
|
uuid: mod.contentId,
|
|
136
206
|
file: path,
|
|
137
|
-
field:
|
|
207
|
+
field: "contentId",
|
|
138
208
|
});
|
|
139
209
|
}
|
|
140
210
|
}
|
|
@@ -146,33 +216,17 @@ export function processContent(files) {
|
|
|
146
216
|
// For split modules, map the file path to the parent slug
|
|
147
217
|
filePathToSlug.set(path, result.modules[0].parentSlug);
|
|
148
218
|
}
|
|
149
|
-
// Collect section-level id:: fields from
|
|
219
|
+
// Collect section-level id:: fields from Lens sections (inline lenses).
|
|
220
|
+
// Referenced lenses (with source::) get their id from the lens file itself.
|
|
150
221
|
if (result.modules.length > 0) {
|
|
151
222
|
const rawParse = parseModule(content, path);
|
|
152
223
|
if (rawParse.module) {
|
|
153
|
-
|
|
154
|
-
if (section.type === 'page' && !section.fields.id) {
|
|
155
|
-
errors.push({
|
|
156
|
-
file: path,
|
|
157
|
-
line: section.line,
|
|
158
|
-
message: `Page section '${section.title}' is missing required id:: field`,
|
|
159
|
-
suggestion: 'Add an id:: field with a UUID (e.g., id:: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)',
|
|
160
|
-
severity: 'error',
|
|
161
|
-
});
|
|
162
|
-
}
|
|
163
|
-
if (section.type === 'page' && section.fields.id) {
|
|
164
|
-
uuidEntries.push({
|
|
165
|
-
uuid: section.fields.id,
|
|
166
|
-
file: path,
|
|
167
|
-
field: 'section id',
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
}
|
|
224
|
+
collectInlineLensUUIDs(rawParse.module.sections, path, uuidEntries, errors);
|
|
171
225
|
}
|
|
172
226
|
}
|
|
173
227
|
errors.push(...result.errors);
|
|
174
228
|
}
|
|
175
|
-
else if (path.startsWith(
|
|
229
|
+
else if (path.startsWith("courses/")) {
|
|
176
230
|
const result = parseCourse(content, path);
|
|
177
231
|
if (result.course) {
|
|
178
232
|
courses.push(result.course);
|
|
@@ -180,35 +234,38 @@ export function processContent(files) {
|
|
|
180
234
|
}
|
|
181
235
|
errors.push(...result.errors);
|
|
182
236
|
}
|
|
183
|
-
else if (path.startsWith(
|
|
237
|
+
else if (path.startsWith("Learning Outcomes/") ||
|
|
238
|
+
path.includes("/Learning Outcomes/")) {
|
|
184
239
|
// Fully validate Learning Outcome (structure, fields, wikilink syntax)
|
|
185
240
|
const result = parseLearningOutcome(content, path);
|
|
186
241
|
errors.push(...result.errors);
|
|
187
242
|
// Check that referenced lens files exist
|
|
188
243
|
if (result.learningOutcome) {
|
|
244
|
+
collectLearningOutcomeTestUUIDs(result.learningOutcome, path, uuidEntries);
|
|
189
245
|
for (const lensRef of result.learningOutcome.lenses) {
|
|
190
246
|
const lensPath = findFileWithExtension(lensRef.resolvedPath, files);
|
|
191
247
|
if (!lensPath) {
|
|
192
248
|
// Find similar files to suggest
|
|
193
|
-
const similarFiles = findSimilarFiles(lensRef.resolvedPath, files,
|
|
194
|
-
const suggestion = formatSuggestion(similarFiles, path) ??
|
|
249
|
+
const similarFiles = findSimilarFiles(lensRef.resolvedPath, files, "Lenses");
|
|
250
|
+
const suggestion = formatSuggestion(similarFiles, path) ??
|
|
251
|
+
"Check the file path in the wiki-link";
|
|
195
252
|
errors.push({
|
|
196
253
|
file: path,
|
|
197
254
|
message: `Referenced lens file not found: ${lensRef.resolvedPath}`,
|
|
198
255
|
suggestion,
|
|
199
|
-
severity:
|
|
256
|
+
severity: "error",
|
|
200
257
|
});
|
|
201
258
|
continue;
|
|
202
259
|
}
|
|
203
260
|
// Check tier violation (LO → Lens)
|
|
204
|
-
const parentTier = tierMap.get(path) ??
|
|
205
|
-
const childTier = tierMap.get(lensPath) ??
|
|
206
|
-
const violation = checkTierViolation(path, parentTier, lensPath, childTier,
|
|
261
|
+
const parentTier = tierMap.get(path) ?? "production";
|
|
262
|
+
const childTier = tierMap.get(lensPath) ?? "production";
|
|
263
|
+
const violation = checkTierViolation(path, parentTier, lensPath, childTier, "lens");
|
|
207
264
|
if (violation) {
|
|
208
265
|
errors.push(violation);
|
|
209
266
|
continue;
|
|
210
267
|
}
|
|
211
|
-
if (childTier ===
|
|
268
|
+
if (childTier === "ignored") {
|
|
212
269
|
continue;
|
|
213
270
|
}
|
|
214
271
|
}
|
|
@@ -218,11 +275,11 @@ export function processContent(files) {
|
|
|
218
275
|
uuidEntries.push({
|
|
219
276
|
uuid: result.learningOutcome.id,
|
|
220
277
|
file: path,
|
|
221
|
-
field:
|
|
278
|
+
field: "id",
|
|
222
279
|
});
|
|
223
280
|
}
|
|
224
281
|
}
|
|
225
|
-
else if (path.startsWith(
|
|
282
|
+
else if (path.startsWith("Lenses/") || path.includes("/Lenses/")) {
|
|
226
283
|
// Fully validate Lens (structure, segments, fields)
|
|
227
284
|
const result = parseLens(content, path);
|
|
228
285
|
errors.push(...result.errors);
|
|
@@ -231,7 +288,7 @@ export function processContent(files) {
|
|
|
231
288
|
uuidEntries.push({
|
|
232
289
|
uuid: result.lens.id,
|
|
233
290
|
file: path,
|
|
234
|
-
field:
|
|
291
|
+
field: "id",
|
|
235
292
|
});
|
|
236
293
|
}
|
|
237
294
|
// Flatten lens as standalone module (pass pre-parsed lens to avoid re-parsing)
|
|
@@ -245,32 +302,65 @@ export function processContent(files) {
|
|
|
245
302
|
errors.push(...lensModuleResult.errors);
|
|
246
303
|
}
|
|
247
304
|
}
|
|
248
|
-
else if (path.endsWith(
|
|
305
|
+
else if (path.endsWith(".timestamps.json")) {
|
|
249
306
|
const tsErrors = validateTimestamps(content, path);
|
|
250
307
|
errors.push(...tsErrors);
|
|
251
308
|
}
|
|
252
|
-
else if (path.startsWith(
|
|
309
|
+
else if (path.startsWith("articles/") || path.includes("/articles/")) {
|
|
253
310
|
const result = parseArticle(content, path);
|
|
254
311
|
errors.push(...result.errors);
|
|
255
312
|
if (result.article) {
|
|
256
|
-
|
|
313
|
+
// Only carry the flag when set true, so URL entries (and thus the
|
|
314
|
+
// serialized output / Python parity) are unchanged for normal articles.
|
|
315
|
+
const overrideFlag = result.article.allowUnreachableUrl
|
|
316
|
+
? { allowUnreachable: true }
|
|
317
|
+
: {};
|
|
318
|
+
urlsToValidate.push({
|
|
319
|
+
url: result.article.sourceUrl,
|
|
320
|
+
file: path,
|
|
321
|
+
line: 2,
|
|
322
|
+
label: "source_url",
|
|
323
|
+
...overrideFlag,
|
|
324
|
+
});
|
|
257
325
|
for (const img of result.article.imageUrls) {
|
|
258
|
-
urlsToValidate.push({
|
|
326
|
+
urlsToValidate.push({
|
|
327
|
+
url: img.url,
|
|
328
|
+
file: path,
|
|
329
|
+
line: img.line,
|
|
330
|
+
label: "Image URL",
|
|
331
|
+
...overrideFlag,
|
|
332
|
+
});
|
|
259
333
|
}
|
|
260
334
|
}
|
|
261
335
|
}
|
|
262
|
-
else if (path.startsWith(
|
|
336
|
+
else if (path.startsWith("video_transcripts/") ||
|
|
337
|
+
path.includes("/video_transcripts/")) {
|
|
263
338
|
const result = parseVideoTranscript(content, path);
|
|
264
339
|
errors.push(...result.errors);
|
|
265
340
|
if (result.transcript) {
|
|
266
|
-
urlsToValidate.push({
|
|
341
|
+
urlsToValidate.push({
|
|
342
|
+
url: result.transcript.url,
|
|
343
|
+
file: path,
|
|
344
|
+
line: 2,
|
|
345
|
+
label: "url",
|
|
346
|
+
...(result.transcript.allowUnreachableUrl
|
|
347
|
+
? { allowUnreachable: true }
|
|
348
|
+
: {}),
|
|
349
|
+
});
|
|
267
350
|
}
|
|
268
351
|
}
|
|
269
352
|
else {
|
|
270
353
|
// File didn't match any known directory pattern — check for near-misses via Levenshtein distance
|
|
271
|
-
const dir = path.split(
|
|
272
|
-
const VALID_DIRS = [
|
|
273
|
-
|
|
354
|
+
const dir = path.split("/")[0];
|
|
355
|
+
const VALID_DIRS = [
|
|
356
|
+
"modules",
|
|
357
|
+
"courses",
|
|
358
|
+
"articles",
|
|
359
|
+
"Lenses",
|
|
360
|
+
"video_transcripts",
|
|
361
|
+
"Learning Outcomes",
|
|
362
|
+
];
|
|
363
|
+
let closest = "";
|
|
274
364
|
let minDist = Infinity;
|
|
275
365
|
for (const valid of VALID_DIRS) {
|
|
276
366
|
const dist = levenshtein(dir.toLowerCase(), valid.toLowerCase());
|
|
@@ -286,18 +376,38 @@ export function processContent(files) {
|
|
|
286
376
|
file: path,
|
|
287
377
|
message: `File in directory '${dir}/' not recognized as content`,
|
|
288
378
|
suggestion: `Did you mean '${closest}/'?`,
|
|
289
|
-
severity:
|
|
379
|
+
severity: "warning",
|
|
290
380
|
});
|
|
291
381
|
}
|
|
292
382
|
}
|
|
293
383
|
}
|
|
384
|
+
// Build contentId → moduleSlug mapping and populate moduleSlug in cross-module card links.
|
|
385
|
+
// Prefer real modules over standalone lenses (lens/ prefix) — a lens that appears in
|
|
386
|
+
// both a module and as a standalone should map to the module.
|
|
387
|
+
const contentIdToModuleSlug = new Map();
|
|
388
|
+
for (const mod of modules) {
|
|
389
|
+
if (mod.slug.startsWith("lens/"))
|
|
390
|
+
continue; // Skip standalone lenses
|
|
391
|
+
for (const section of mod.sections) {
|
|
392
|
+
if (section.contentId) {
|
|
393
|
+
contentIdToModuleSlug.set(section.contentId, mod.slug);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
for (const mod of modules) {
|
|
398
|
+
populateCardModuleSlugs(mod.sections, contentIdToModuleSlug);
|
|
399
|
+
resolveInlineLensModuleSlugs(mod.sections, contentIdToModuleSlug);
|
|
400
|
+
}
|
|
401
|
+
// Card-to-module membership check. Runs after card link resolution above so
|
|
402
|
+
// every card has its data-lens-card JSON populated, including allowExternal.
|
|
403
|
+
errors.push(...validateCardModuleMembership(modules));
|
|
294
404
|
// Resolve course module paths to frontmatter slugs.
|
|
295
405
|
// Use filePathToSlug (built during module parsing) instead of inverting slugToPath,
|
|
296
406
|
// because slugToPath loses entries when duplicate slugs exist.
|
|
297
407
|
for (const course of courses) {
|
|
298
|
-
const courseFile = courseSlugToFile.get(course.slug) ??
|
|
408
|
+
const courseFile = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
299
409
|
for (const item of course.progression) {
|
|
300
|
-
if (item.type ===
|
|
410
|
+
if (item.type === "module" && item.path) {
|
|
301
411
|
// Resolve wikilink path relative to the course file
|
|
302
412
|
const resolved = resolveWikilinkPath(item.path, courseFile);
|
|
303
413
|
const actualFile = findFileWithExtension(resolved, files);
|
|
@@ -306,10 +416,10 @@ export function processContent(files) {
|
|
|
306
416
|
}
|
|
307
417
|
else {
|
|
308
418
|
// Try matching just the filename stem against module file stems
|
|
309
|
-
const stem = item.path.split(
|
|
419
|
+
const stem = item.path.split("/").pop() ?? item.path;
|
|
310
420
|
let matched = false;
|
|
311
421
|
for (const [filePath, slug] of filePathToSlug.entries()) {
|
|
312
|
-
const fileStem = filePath.replace(/\.md$/,
|
|
422
|
+
const fileStem = filePath.replace(/\.md$/, "").split("/").pop() ?? "";
|
|
313
423
|
if (fileStem === stem) {
|
|
314
424
|
item.slug = slug;
|
|
315
425
|
matched = true;
|
|
@@ -320,8 +430,8 @@ export function processContent(files) {
|
|
|
320
430
|
errors.push({
|
|
321
431
|
file: courseFile,
|
|
322
432
|
message: `Module reference could not be resolved: "${item.path}"`,
|
|
323
|
-
suggestion:
|
|
324
|
-
severity:
|
|
433
|
+
suggestion: "Check that the wikilink path points to an existing module file",
|
|
434
|
+
severity: "error",
|
|
325
435
|
});
|
|
326
436
|
}
|
|
327
437
|
}
|
|
@@ -330,11 +440,13 @@ export function processContent(files) {
|
|
|
330
440
|
}
|
|
331
441
|
}
|
|
332
442
|
// Remove unresolved module items (no slug after resolution)
|
|
333
|
-
course.progression = course.progression.filter(item => item.type !==
|
|
443
|
+
course.progression = course.progression.filter((item) => item.type !== "module" || item.slug !== undefined);
|
|
334
444
|
// Expand split modules: replace parent slug with child submodule slugs
|
|
335
445
|
const expanded = [];
|
|
336
446
|
for (const item of course.progression) {
|
|
337
|
-
if (item.type ===
|
|
447
|
+
if (item.type === "module" &&
|
|
448
|
+
item.slug &&
|
|
449
|
+
parentSlugToChildren.has(item.slug)) {
|
|
338
450
|
const children = parentSlugToChildren.get(item.slug);
|
|
339
451
|
for (const childSlug of children) {
|
|
340
452
|
expanded.push({ ...item, slug: childSlug });
|
|
@@ -352,19 +464,48 @@ export function processContent(files) {
|
|
|
352
464
|
if (!coursePath)
|
|
353
465
|
continue;
|
|
354
466
|
for (const item of course.progression) {
|
|
355
|
-
if (item.type !==
|
|
467
|
+
if (item.type !== "module" || !item.slug)
|
|
356
468
|
continue;
|
|
357
469
|
const modulePath = slugToPath.get(item.slug);
|
|
358
470
|
if (modulePath && tierMap.has(modulePath)) {
|
|
359
|
-
const parentTier = tierMap.get(coursePath) ??
|
|
360
|
-
const childTier = tierMap.get(modulePath) ??
|
|
361
|
-
const violation = checkTierViolation(coursePath, parentTier, modulePath, childTier,
|
|
471
|
+
const parentTier = tierMap.get(coursePath) ?? "production";
|
|
472
|
+
const childTier = tierMap.get(modulePath) ?? "production";
|
|
473
|
+
const violation = checkTierViolation(coursePath, parentTier, modulePath, childTier, "module");
|
|
362
474
|
if (violation) {
|
|
363
475
|
errors.push(violation);
|
|
364
476
|
}
|
|
365
477
|
}
|
|
366
478
|
}
|
|
367
479
|
}
|
|
480
|
+
// Validate course slug alias collisions
|
|
481
|
+
{
|
|
482
|
+
// Map of all course slugs (primary + aliases) -> source file
|
|
483
|
+
const allCourseSlugs = new Map(); // slug -> file
|
|
484
|
+
// Register primary slugs first
|
|
485
|
+
for (const course of courses) {
|
|
486
|
+
const file = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
487
|
+
allCourseSlugs.set(course.slug, file);
|
|
488
|
+
}
|
|
489
|
+
// Check each alias against all known slugs
|
|
490
|
+
for (const course of courses) {
|
|
491
|
+
const file = courseSlugToFile.get(course.slug) ?? "courses/";
|
|
492
|
+
const tier = tierMap.get(file) ?? "production";
|
|
493
|
+
for (const alias of course.slugAliases ?? []) {
|
|
494
|
+
const existing = allCourseSlugs.get(alias);
|
|
495
|
+
if (existing) {
|
|
496
|
+
errors.push({
|
|
497
|
+
file,
|
|
498
|
+
message: `Course slug alias '${alias}' collides with ${existing === file ? "its own primary slug" : `slug in ${existing}`}`,
|
|
499
|
+
suggestion: `Choose a different alias or remove the conflicting slug`,
|
|
500
|
+
severity: tier === "wip" ? "warning" : "error",
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
else {
|
|
504
|
+
allCourseSlugs.set(alias, file);
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
}
|
|
368
509
|
// Validate all collected UUIDs
|
|
369
510
|
const uuidValidation = validateUuids(uuidEntries);
|
|
370
511
|
errors.push(...uuidValidation.errors);
|
|
@@ -372,29 +513,30 @@ export function processContent(files) {
|
|
|
372
513
|
const duplicateSlugErrors = detectDuplicateSlugs(slugEntries);
|
|
373
514
|
errors.push(...duplicateSlugErrors);
|
|
374
515
|
// Validate video transcript / timestamps.json pairing
|
|
375
|
-
const transcriptPaths = [...files.keys()].filter(p => (p.startsWith(
|
|
376
|
-
p.
|
|
377
|
-
|
|
378
|
-
|
|
516
|
+
const transcriptPaths = [...files.keys()].filter((p) => (p.startsWith("video_transcripts/") ||
|
|
517
|
+
p.includes("/video_transcripts/")) &&
|
|
518
|
+
p.endsWith(".md") &&
|
|
519
|
+
tierMap.get(p) !== "ignored");
|
|
520
|
+
const timestampPaths = new Set([...files.keys()].filter((p) => p.endsWith(".timestamps.json")));
|
|
379
521
|
for (const mdPath of transcriptPaths) {
|
|
380
|
-
const expectedTs = mdPath.replace(/\.md$/,
|
|
522
|
+
const expectedTs = mdPath.replace(/\.md$/, ".timestamps.json");
|
|
381
523
|
if (!timestampPaths.has(expectedTs)) {
|
|
382
524
|
errors.push({
|
|
383
525
|
file: mdPath,
|
|
384
526
|
message: `Missing timestamps.json: expected ${expectedTs}`,
|
|
385
|
-
severity:
|
|
527
|
+
severity: "error",
|
|
386
528
|
});
|
|
387
529
|
}
|
|
388
530
|
}
|
|
389
531
|
for (const tsPath of timestampPaths) {
|
|
390
|
-
const expectedMd = tsPath.replace(/\.timestamps\.json$/,
|
|
391
|
-
if (tierMap.get(expectedMd) ===
|
|
532
|
+
const expectedMd = tsPath.replace(/\.timestamps\.json$/, ".md");
|
|
533
|
+
if (tierMap.get(expectedMd) === "ignored")
|
|
392
534
|
continue;
|
|
393
535
|
if (!files.has(expectedMd)) {
|
|
394
536
|
errors.push({
|
|
395
537
|
file: tsPath,
|
|
396
538
|
message: `Orphaned timestamps file: no matching .md transcript found`,
|
|
397
|
-
severity:
|
|
539
|
+
severity: "warning",
|
|
398
540
|
});
|
|
399
541
|
}
|
|
400
542
|
}
|
|
@@ -405,9 +547,12 @@ export function processContent(files) {
|
|
|
405
547
|
for (const error of errors) {
|
|
406
548
|
if (!error.category) {
|
|
407
549
|
const tier = tierMap.get(error.file);
|
|
408
|
-
error.category = tier ===
|
|
550
|
+
error.category = tier === "wip" ? "wip" : "production";
|
|
409
551
|
}
|
|
410
552
|
}
|
|
411
|
-
|
|
553
|
+
// The same file is validated once per pass (standalone + once per module
|
|
554
|
+
// that flattens it), so identical errors are emitted multiple times. Collapse
|
|
555
|
+
// them so each problem is reported once.
|
|
556
|
+
return { modules, courses, errors: dedupeErrors(errors), urlsToValidate };
|
|
412
557
|
}
|
|
413
558
|
//# sourceMappingURL=index.js.map
|