lens-content-processor 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundler/video.js +4 -4
- package/dist/bundler/video.js.map +1 -1
- package/dist/cli.d.ts +1 -2
- package/dist/cli.js +11 -9
- package/dist/cli.js.map +1 -1
- package/dist/content-schema.d.ts +34 -0
- package/dist/content-schema.js +62 -0
- package/dist/content-schema.js.map +1 -0
- package/dist/flattener/index.d.ts +21 -1
- package/dist/flattener/index.js +316 -33
- package/dist/flattener/index.js.map +1 -1
- package/dist/fs/read-vault.d.ts +1 -4
- package/dist/fs/read-vault.js +4 -8
- package/dist/fs/read-vault.js.map +1 -1
- package/dist/index.d.ts +25 -2
- package/dist/index.js +231 -6
- package/dist/index.js.map +1 -1
- package/dist/parser/article.d.ts +16 -0
- package/dist/parser/article.js +70 -0
- package/dist/parser/article.js.map +1 -0
- package/dist/parser/course.d.ts +2 -2
- package/dist/parser/course.js +18 -35
- package/dist/parser/course.js.map +1 -1
- package/dist/parser/frontmatter.js +2 -0
- package/dist/parser/frontmatter.js.map +1 -1
- package/dist/parser/learning-outcome.d.ts +4 -2
- package/dist/parser/learning-outcome.js +65 -23
- package/dist/parser/learning-outcome.js.map +1 -1
- package/dist/parser/lens.d.ts +38 -1
- package/dist/parser/lens.js +173 -18
- package/dist/parser/lens.js.map +1 -1
- package/dist/parser/module.d.ts +11 -5
- package/dist/parser/module.js +172 -76
- package/dist/parser/module.js.map +1 -1
- package/dist/parser/sections.js +116 -13
- package/dist/parser/sections.js.map +1 -1
- package/dist/parser/video-transcript.d.ts +11 -0
- package/dist/parser/video-transcript.js +39 -0
- package/dist/parser/video-transcript.js.map +1 -0
- package/dist/parser/wikilink.d.ts +2 -0
- package/dist/parser/wikilink.js +88 -24
- package/dist/parser/wikilink.js.map +1 -1
- package/dist/utils/slug.d.ts +5 -0
- package/dist/utils/slug.js +16 -0
- package/dist/utils/slug.js.map +1 -0
- package/dist/validator/chat-precedence.d.ts +9 -0
- package/dist/validator/chat-precedence.js +32 -0
- package/dist/validator/chat-precedence.js.map +1 -0
- package/dist/validator/duplicates.d.ts +13 -0
- package/dist/validator/duplicates.js +27 -0
- package/dist/validator/duplicates.js.map +1 -0
- package/dist/validator/field-typos.d.ts +11 -0
- package/dist/validator/field-typos.js +36 -28
- package/dist/validator/field-typos.js.map +1 -1
- package/dist/validator/field-values.d.ts +14 -0
- package/dist/validator/field-values.js +61 -7
- package/dist/validator/field-values.js.map +1 -1
- package/dist/validator/output-integrity.d.ts +7 -0
- package/dist/validator/output-integrity.js +57 -0
- package/dist/validator/output-integrity.js.map +1 -0
- package/dist/validator/segment-fields.js +1 -15
- package/dist/validator/segment-fields.js.map +1 -1
- package/dist/validator/tier.d.ts +18 -0
- package/dist/validator/tier.js +74 -0
- package/dist/validator/tier.js.map +1 -0
- package/dist/validator/timestamps.d.ts +2 -0
- package/dist/validator/timestamps.js +91 -0
- package/dist/validator/timestamps.js.map +1 -0
- package/dist/validator/url-reachability.d.ts +8 -0
- package/dist/validator/url-reachability.js +60 -0
- package/dist/validator/url-reachability.js.map +1 -0
- package/dist/validator/validate-frontmatter.d.ts +9 -0
- package/dist/validator/validate-frontmatter.js +53 -0
- package/dist/validator/validate-frontmatter.js.map +1 -0
- package/package.json +5 -5
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ContentError } from '../index.js';
|
|
2
|
+
export type ContentTier = 'production' | 'wip' | 'ignored';
|
|
3
|
+
/**
|
|
4
|
+
* Determine content tier from frontmatter tags.
|
|
5
|
+
* - tags containing 'validator-ignore' → 'ignored'
|
|
6
|
+
* - tags containing 'wip' → 'wip'
|
|
7
|
+
* - otherwise → 'production'
|
|
8
|
+
*/
|
|
9
|
+
export declare function getTierFromFrontmatter(frontmatter: Record<string, unknown>): ContentTier;
|
|
10
|
+
/**
|
|
11
|
+
* Build a tier map for all .md files by parsing their frontmatter tags.
|
|
12
|
+
*/
|
|
13
|
+
export declare function buildTierMap(files: Map<string, string>): Map<string, ContentTier>;
|
|
14
|
+
/**
|
|
15
|
+
* Check for tier violations when a parent references a child.
|
|
16
|
+
* Returns a ContentError if the reference violates tier rules, null otherwise.
|
|
17
|
+
*/
|
|
18
|
+
export declare function checkTierViolation(parentPath: string, parentTier: ContentTier, childPath: string, childTier: ContentTier, childLabel: string, line?: number): ContentError | null;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { parseFrontmatter } from '../parser/frontmatter.js';
|
|
2
|
+
/**
|
|
3
|
+
* Determine content tier from frontmatter tags.
|
|
4
|
+
* - tags containing 'validator-ignore' → 'ignored'
|
|
5
|
+
* - tags containing 'wip' → 'wip'
|
|
6
|
+
* - otherwise → 'production'
|
|
7
|
+
*/
|
|
8
|
+
export function getTierFromFrontmatter(frontmatter) {
|
|
9
|
+
const tags = frontmatter.tags;
|
|
10
|
+
if (!Array.isArray(tags))
|
|
11
|
+
return 'production';
|
|
12
|
+
const normalized = tags.map((t) => String(t).toLowerCase());
|
|
13
|
+
if (normalized.includes('validator-ignore'))
|
|
14
|
+
return 'ignored';
|
|
15
|
+
if (normalized.includes('wip') || normalized.includes('work-in-progress'))
|
|
16
|
+
return 'wip';
|
|
17
|
+
return 'production';
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Build a tier map for all .md files by parsing their frontmatter tags.
|
|
21
|
+
*/
|
|
22
|
+
export function buildTierMap(files) {
|
|
23
|
+
const tierMap = new Map();
|
|
24
|
+
for (const [path, content] of files.entries()) {
|
|
25
|
+
if (!path.endsWith('.md'))
|
|
26
|
+
continue;
|
|
27
|
+
const { frontmatter } = parseFrontmatter(content, path);
|
|
28
|
+
tierMap.set(path, getTierFromFrontmatter(frontmatter));
|
|
29
|
+
}
|
|
30
|
+
return tierMap;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Check for tier violations when a parent references a child.
|
|
34
|
+
* Returns a ContentError if the reference violates tier rules, null otherwise.
|
|
35
|
+
*/
|
|
36
|
+
export function checkTierViolation(parentPath, parentTier, childPath, childTier, childLabel, line) {
|
|
37
|
+
// Ignored parents are never processed (filtered upstream)
|
|
38
|
+
if (parentTier === 'ignored')
|
|
39
|
+
return null;
|
|
40
|
+
// Production parent → child must be production
|
|
41
|
+
if (parentTier === 'production' && childTier === 'wip') {
|
|
42
|
+
return {
|
|
43
|
+
file: parentPath,
|
|
44
|
+
line,
|
|
45
|
+
message: `References WIP ${childLabel}: ${childPath}`,
|
|
46
|
+
suggestion: `Remove the WIP tag from ${childPath}, or add tags: [wip] to this file`,
|
|
47
|
+
severity: 'error',
|
|
48
|
+
category: 'production',
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
if (parentTier === 'production' && childTier === 'ignored') {
|
|
52
|
+
return {
|
|
53
|
+
file: parentPath,
|
|
54
|
+
line,
|
|
55
|
+
message: `References ignored ${childLabel}: ${childPath}`,
|
|
56
|
+
suggestion: `Remove the validator-ignore tag from ${childPath}, or add tags: [wip] to this file`,
|
|
57
|
+
severity: 'error',
|
|
58
|
+
category: 'production',
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
// WIP parent → child must not be ignored
|
|
62
|
+
if (parentTier === 'wip' && childTier === 'ignored') {
|
|
63
|
+
return {
|
|
64
|
+
file: parentPath,
|
|
65
|
+
line,
|
|
66
|
+
message: `References ignored ${childLabel}: ${childPath}`,
|
|
67
|
+
suggestion: `Remove the validator-ignore tag from ${childPath}`,
|
|
68
|
+
severity: 'error',
|
|
69
|
+
category: 'wip',
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=tier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tier.js","sourceRoot":"","sources":["../../src/validator/tier.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,WAAoC;IACzE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;IAC9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,YAAY,CAAC;IAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;IACrE,IAAI,UAAU,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9D,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QAAE,OAAO,KAAK,CAAC;IACxF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAAC,KAA0B;IACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QACpC,MAAM,EAAE,WAAW,EAAE,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,sBAAsB,CAAC,WAAW,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAkB,EAClB,UAAuB,EACvB,SAAiB,EACjB,SAAsB,EACtB,UAAkB,EAClB,IAAa;IAEb,0DAA0D;IAC1D,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAE1C,+CAA+C;IAC/C,IAAI,UAAU,KAAK,YAAY,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACvD,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,IAAI;YACJ,OAAO,EAAE,kBAAkB,UAAU,KAAK,SAAS,EAAE;YACrD,UAAU,EAAE,2BAA2B,SAAS,mCAAmC;YACnF,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,YAAY;SACvB,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,KAAK,YAAY,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC3D,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,IAAI;YACJ,OAAO,EAAE,sBAAsB,UAAU,KAAK,SAAS,EAAE;YACzD,UAAU,EAAE,wCAAwC,SAAS,mCAAmC;YAChG,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,YAAY;SACvB,CAAC;IACJ,CAAC;IAED,yCAAyC;IACzC,IAAI,UAAU,KAAK,KAAK,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACpD,OAAO;YACL,IAAI,EAAE,UAAU;YAChB,IAAI;YACJ,OAAO,EAAE,sBAAsB,UAAU,KAAK,SAAS,EAAE;YACzD,UAAU,EAAE,wCAAwC,SAAS,EAAE;YAC/D,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,KAAK;SAChB,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { parseTimestamp } from '../bundler/video.js';
|
|
2
|
+
export function validateTimestamps(content, file) {
|
|
3
|
+
const errors = [];
|
|
4
|
+
let parsed;
|
|
5
|
+
try {
|
|
6
|
+
parsed = JSON.parse(content);
|
|
7
|
+
}
|
|
8
|
+
catch (e) {
|
|
9
|
+
errors.push({
|
|
10
|
+
file,
|
|
11
|
+
line: 1,
|
|
12
|
+
message: `Invalid JSON: ${e instanceof Error ? e.message : String(e)}`,
|
|
13
|
+
severity: 'error',
|
|
14
|
+
});
|
|
15
|
+
return errors;
|
|
16
|
+
}
|
|
17
|
+
if (!Array.isArray(parsed)) {
|
|
18
|
+
errors.push({
|
|
19
|
+
file,
|
|
20
|
+
line: 1,
|
|
21
|
+
message: 'Timestamps file must contain a JSON array',
|
|
22
|
+
severity: 'error',
|
|
23
|
+
});
|
|
24
|
+
return errors;
|
|
25
|
+
}
|
|
26
|
+
if (parsed.length === 0) {
|
|
27
|
+
errors.push({
|
|
28
|
+
file,
|
|
29
|
+
line: 1,
|
|
30
|
+
message: 'Timestamps array is empty',
|
|
31
|
+
severity: 'warning',
|
|
32
|
+
});
|
|
33
|
+
return errors;
|
|
34
|
+
}
|
|
35
|
+
let prevSeconds = null;
|
|
36
|
+
for (let i = 0; i < parsed.length; i++) {
|
|
37
|
+
const entry = parsed[i];
|
|
38
|
+
if (typeof entry.text !== 'string' && entry.text !== undefined) {
|
|
39
|
+
errors.push({
|
|
40
|
+
file,
|
|
41
|
+
message: `Entry ${i}: 'text' must be a string`,
|
|
42
|
+
severity: 'error',
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
else if (entry.text === undefined) {
|
|
46
|
+
errors.push({
|
|
47
|
+
file,
|
|
48
|
+
message: `Entry ${i}: missing required field 'text'`,
|
|
49
|
+
severity: 'error',
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
if (entry.start === undefined) {
|
|
53
|
+
errors.push({
|
|
54
|
+
file,
|
|
55
|
+
message: `Entry ${i}: missing required field 'start'`,
|
|
56
|
+
severity: 'error',
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
else if (typeof entry.start === 'string') {
|
|
60
|
+
const seconds = parseTimestamp(entry.start);
|
|
61
|
+
if (seconds === null) {
|
|
62
|
+
errors.push({
|
|
63
|
+
file,
|
|
64
|
+
message: `Entry ${i}: invalid timestamp format '${entry.start}'`,
|
|
65
|
+
suggestion: 'Use format M:SS.ms (e.g., 0:01.32)',
|
|
66
|
+
severity: 'error',
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
if (prevSeconds !== null && seconds < prevSeconds) {
|
|
71
|
+
errors.push({
|
|
72
|
+
file,
|
|
73
|
+
message: `Entry ${i}: non-monotonic timestamp '${entry.start}' (previous was later)`,
|
|
74
|
+
severity: 'warning',
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
prevSeconds = seconds;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
errors.push({
|
|
82
|
+
file,
|
|
83
|
+
message: `Entry ${i}: 'start' must be a string, got ${typeof entry.start}`,
|
|
84
|
+
suggestion: "Use string format like '0:01.32' instead of a number",
|
|
85
|
+
severity: 'error',
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return errors;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=timestamps.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timestamps.js","sourceRoot":"","sources":["../../src/validator/timestamps.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,IAAY;IAC9D,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,IAAI,EAAE,CAAC;YACP,OAAO,EAAE,iBAAiB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;YACtE,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,IAAI,EAAE,CAAC;YACP,OAAO,EAAE,2CAA2C;YACpD,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI;YACJ,IAAI,EAAE,CAAC;YACP,OAAO,EAAE,2BAA2B;YACpC,QAAQ,EAAE,SAAS;SACpB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,WAAW,GAAkB,IAAI,CAAC;IAEtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAA4B,CAAC;QAEnD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,SAAS,CAAC,2BAA2B;gBAC9C,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,SAAS,CAAC,iCAAiC;gBACpD,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,SAAS,CAAC,kCAAkC;gBACrD,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI;oBACJ,OAAO,EAAE,SAAS,CAAC,+BAA+B,KAAK,CAAC,KAAK,GAAG;oBAChE,UAAU,EAAE,oCAAoC;oBAChD,QAAQ,EAAE,OAAO;iBAClB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,IAAI,WAAW,KAAK,IAAI,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;oBAClD,MAAM,CAAC,IAAI,CAAC;wBACV,IAAI;wBACJ,OAAO,EAAE,SAAS,CAAC,8BAA8B,KAAK,CAAC,KAAK,wBAAwB;wBACpF,QAAQ,EAAE,SAAS;qBACpB,CAAC,CAAC;gBACL,CAAC;gBACD,WAAW,GAAG,OAAO,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,OAAO,EAAE,SAAS,CAAC,mCAAmC,OAAO,KAAK,CAAC,KAAK,EAAE;gBAC1E,UAAU,EAAE,sDAAsD;gBAClE,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
const CONCURRENCY = 10;
|
|
2
|
+
const TIMEOUT_MS = 15_000;
|
|
3
|
+
// Status codes that mean "server is alive" even though response.ok is false
|
|
4
|
+
const REACHABLE_STATUSES = new Set([429]);
|
|
5
|
+
async function fetchWithTimeout(url, method, signal) {
|
|
6
|
+
return fetch(url, { method, signal });
|
|
7
|
+
}
|
|
8
|
+
async function checkUrl(entry) {
|
|
9
|
+
try {
|
|
10
|
+
const controller = new AbortController();
|
|
11
|
+
const timeout = setTimeout(() => controller.abort(), TIMEOUT_MS);
|
|
12
|
+
let response = await fetchWithTimeout(entry.url, 'HEAD', controller.signal);
|
|
13
|
+
// Some servers reject HEAD — retry with GET
|
|
14
|
+
if (response.status === 405) {
|
|
15
|
+
response = await fetchWithTimeout(entry.url, 'GET', controller.signal);
|
|
16
|
+
}
|
|
17
|
+
clearTimeout(timeout);
|
|
18
|
+
if (!response.ok && !REACHABLE_STATUSES.has(response.status)) {
|
|
19
|
+
return {
|
|
20
|
+
file: entry.file,
|
|
21
|
+
line: entry.line,
|
|
22
|
+
message: `${entry.label} returned HTTP ${response.status}: ${entry.url}`,
|
|
23
|
+
severity: 'warning',
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (error instanceof DOMException && error.name === 'AbortError') {
|
|
30
|
+
return {
|
|
31
|
+
file: entry.file,
|
|
32
|
+
line: entry.line,
|
|
33
|
+
message: `${entry.label} timed out: ${entry.url}`,
|
|
34
|
+
severity: 'warning',
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
file: entry.file,
|
|
39
|
+
line: entry.line,
|
|
40
|
+
message: `${entry.label} unreachable: ${entry.url}`,
|
|
41
|
+
severity: 'warning',
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export async function validateUrls(urls) {
|
|
46
|
+
if (urls.length === 0)
|
|
47
|
+
return [];
|
|
48
|
+
const warnings = [];
|
|
49
|
+
// Process in batches to avoid overwhelming servers/network
|
|
50
|
+
for (let i = 0; i < urls.length; i += CONCURRENCY) {
|
|
51
|
+
const batch = urls.slice(i, i + CONCURRENCY);
|
|
52
|
+
const results = await Promise.all(batch.map(checkUrl));
|
|
53
|
+
for (const result of results) {
|
|
54
|
+
if (result)
|
|
55
|
+
warnings.push(result);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return warnings;
|
|
59
|
+
}
|
|
60
|
+
//# sourceMappingURL=url-reachability.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-reachability.js","sourceRoot":"","sources":["../../src/validator/url-reachability.ts"],"names":[],"mappings":"AAUA,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,4EAA4E;AAC5E,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1C,KAAK,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc,EAAE,MAAmB;IAC9E,OAAO,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,KAAoB;IAC1C,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;QAEjE,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAE5E,4CAA4C;QAC5C,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YAC5B,QAAQ,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACzE,CAAC;QAED,YAAY,CAAC,OAAO,CAAC,CAAC;QAEtB,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC7D,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,kBAAkB,QAAQ,CAAC,MAAM,KAAK,KAAK,CAAC,GAAG,EAAE;gBACxE,QAAQ,EAAE,SAAS;aACpB,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACjE,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,eAAe,KAAK,CAAC,GAAG,EAAE;gBACjD,QAAQ,EAAE,SAAS;aACpB,CAAC;QACJ,CAAC;QACD,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,iBAAiB,KAAK,CAAC,GAAG,EAAE;YACnD,QAAQ,EAAE,SAAS;SACpB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAqB;IAErB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,MAAM,QAAQ,GAAmB,EAAE,CAAC;IAEpC,2DAA2D;IAC3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,MAAM;gBAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ContentError } from '../index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Generic frontmatter validation against the schema for a content type.
|
|
4
|
+
* Checks required fields (present + non-empty) and detects typos.
|
|
5
|
+
*
|
|
6
|
+
* Note: This does NOT handle type-specific validation like slug format
|
|
7
|
+
* or id-must-be-string. Parsers still handle those themselves.
|
|
8
|
+
*/
|
|
9
|
+
export declare function validateFrontmatter(frontmatter: Record<string, unknown>, contentType: string, file: string): ContentError[];
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { CONTENT_SCHEMAS } from '../content-schema.js';
|
|
2
|
+
import { detectFrontmatterTypos, levenshtein } from './field-typos.js';
|
|
3
|
+
/**
|
|
4
|
+
* Generic frontmatter validation against the schema for a content type.
|
|
5
|
+
* Checks required fields (present + non-empty) and detects typos.
|
|
6
|
+
*
|
|
7
|
+
* Note: This does NOT handle type-specific validation like slug format
|
|
8
|
+
* or id-must-be-string. Parsers still handle those themselves.
|
|
9
|
+
*/
|
|
10
|
+
export function validateFrontmatter(frontmatter, contentType, file) {
|
|
11
|
+
const schema = CONTENT_SCHEMAS[contentType];
|
|
12
|
+
if (!schema)
|
|
13
|
+
return [];
|
|
14
|
+
const errors = [];
|
|
15
|
+
// Detect typos in field names
|
|
16
|
+
errors.push(...detectFrontmatterTypos(frontmatter, schema.allFields, file));
|
|
17
|
+
// Check required fields
|
|
18
|
+
for (const field of schema.requiredFields) {
|
|
19
|
+
const value = frontmatter[field];
|
|
20
|
+
if (value === undefined || value === null) {
|
|
21
|
+
// Check if a similar field exists (likely typo or wrong name)
|
|
22
|
+
let suggestion = `Add '${field}' to frontmatter`;
|
|
23
|
+
const presentFields = Object.keys(frontmatter);
|
|
24
|
+
for (const present of presentFields) {
|
|
25
|
+
if (schema.requiredFields.includes(present))
|
|
26
|
+
continue; // skip other required fields
|
|
27
|
+
const dist = levenshtein(field.toLowerCase(), present.toLowerCase());
|
|
28
|
+
if (dist <= 3 || field.toLowerCase().includes(present.toLowerCase()) || present.toLowerCase().includes(field.toLowerCase())) {
|
|
29
|
+
suggestion = `Did you mean '${field}' instead of '${present}'?`;
|
|
30
|
+
break;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
errors.push({
|
|
34
|
+
file,
|
|
35
|
+
line: 2,
|
|
36
|
+
message: `Missing required field: ${field}`,
|
|
37
|
+
suggestion,
|
|
38
|
+
severity: 'error',
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
else if (typeof value === 'string' && value.trim() === '') {
|
|
42
|
+
errors.push({
|
|
43
|
+
file,
|
|
44
|
+
line: 2,
|
|
45
|
+
message: `Field '${field}' must not be empty`,
|
|
46
|
+
suggestion: `Provide a value for '${field}'`,
|
|
47
|
+
severity: 'error',
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return errors;
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=validate-frontmatter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validate-frontmatter.js","sourceRoot":"","sources":["../../src/validator/validate-frontmatter.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,sBAAsB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEvE;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CACjC,WAAoC,EACpC,WAAmB,EACnB,IAAY;IAEZ,MAAM,MAAM,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAEvB,MAAM,MAAM,GAAmB,EAAE,CAAC;IAElC,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,sBAAsB,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC;IAE5E,wBAAwB;IACxB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1C,8DAA8D;YAC9D,IAAI,UAAU,GAAG,QAAQ,KAAK,kBAAkB,CAAC;YACjD,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/C,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;gBACpC,IAAI,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,OAAO,CAAC;oBAAE,SAAS,CAAC,6BAA6B;gBACpF,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;gBACrE,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;oBAC5H,UAAU,GAAG,iBAAiB,KAAK,iBAAiB,OAAO,IAAI,CAAC;oBAChE,MAAM;gBACR,CAAC;YACH,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,IAAI,EAAE,CAAC;gBACP,OAAO,EAAE,2BAA2B,KAAK,EAAE;gBAC3C,UAAU;gBACV,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;aAAM,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;YAC5D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI;gBACJ,IAAI,EAAE,CAAC;gBACP,OAAO,EAAE,UAAU,KAAK,qBAAqB;gBAC7C,UAAU,EAAE,wBAAwB,KAAK,GAAG;gBAC5C,QAAQ,EAAE,OAAO;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "lens-content-processor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "Parser and validator for Lens educational content (Markdown modules, lenses, learning outcomes)",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -34,12 +34,12 @@
|
|
|
34
34
|
"directory": "content_processor"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
|
-
"
|
|
38
|
-
"vitest": "^3.0.0",
|
|
37
|
+
"@types/node": "^22.0.0",
|
|
39
38
|
"tsx": "^4.0.0",
|
|
40
|
-
"
|
|
39
|
+
"typescript": "^5.7.0",
|
|
40
|
+
"vitest": "^3.0.0"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
-
"yaml": "^2.
|
|
43
|
+
"yaml": "^2.8.2"
|
|
44
44
|
}
|
|
45
45
|
}
|