@vibe-agent-toolkit/resources 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/external-link-cache.d.ts +96 -0
- package/dist/external-link-cache.d.ts.map +1 -0
- package/dist/external-link-cache.js +183 -0
- package/dist/external-link-cache.js.map +1 -0
- package/dist/external-link-validator.d.ts +88 -0
- package/dist/external-link-validator.d.ts.map +1 -0
- package/dist/external-link-validator.js +194 -0
- package/dist/external-link-validator.js.map +1 -0
- package/dist/link-parser.js +11 -0
- package/dist/link-parser.js.map +1 -1
- package/dist/resource-registry.d.ts +37 -0
- package/dist/resource-registry.d.ts.map +1 -1
- package/dist/resource-registry.js +107 -1
- package/dist/resource-registry.js.map +1 -1
- package/dist/schemas/project-config.d.ts +210 -0
- package/dist/schemas/project-config.d.ts.map +1 -1
- package/dist/schemas/project-config.js +21 -0
- package/dist/schemas/project-config.js.map +1 -1
- package/dist/schemas/validation-result.d.ts +3 -0
- package/dist/schemas/validation-result.d.ts.map +1 -1
- package/dist/schemas/validation-result.js +3 -0
- package/dist/schemas/validation-result.js.map +1 -1
- package/package.json +3 -2
- package/src/external-link-cache.ts +215 -0
- package/src/external-link-validator.ts +250 -0
- package/src/link-parser.ts +13 -1
- package/src/resource-registry.ts +131 -1
- package/src/schemas/project-config.ts +24 -0
- package/src/schemas/validation-result.ts +3 -0
- package/src/types/markdown-link-check.d.ts +33 -0
|
@@ -7,6 +7,25 @@ import { z } from 'zod';
|
|
|
7
7
|
*/
|
|
8
8
|
export const ValidationModeSchema = z.enum(['strict', 'permissive'])
|
|
9
9
|
.describe('Validation mode for frontmatter schema validation');
|
|
10
|
+
/**
|
|
11
|
+
* External URL validation configuration.
|
|
12
|
+
*
|
|
13
|
+
* Controls how external URLs are validated:
|
|
14
|
+
* - enabled: Whether to check external URLs
|
|
15
|
+
* - timeout: Request timeout in milliseconds (default: 15000)
|
|
16
|
+
* - retryOn429: Whether to retry on rate limit (default: true)
|
|
17
|
+
* - ignorePatterns: Regex patterns for URLs to skip (e.g., '^https://localhost')
|
|
18
|
+
*/
|
|
19
|
+
export const ExternalUrlValidationSchema = z.object({
|
|
20
|
+
enabled: z.boolean().optional()
|
|
21
|
+
.describe('Whether to validate external URLs (default: false)'),
|
|
22
|
+
timeout: z.number().int().positive().optional()
|
|
23
|
+
.describe('Request timeout in milliseconds (default: 15000)'),
|
|
24
|
+
retryOn429: z.boolean().optional()
|
|
25
|
+
.describe('Whether to retry on rate limit (429) (default: true)'),
|
|
26
|
+
ignorePatterns: z.array(z.string()).optional()
|
|
27
|
+
.describe('Regex patterns for URLs to skip validation (e.g., "^https://localhost")'),
|
|
28
|
+
}).describe('External URL validation configuration');
|
|
10
29
|
/**
|
|
11
30
|
* Validation configuration for a collection.
|
|
12
31
|
*/
|
|
@@ -19,6 +38,8 @@ export const CollectionValidationSchema = z.object({
|
|
|
19
38
|
.describe('Whether to validate external URL links (default: false)'),
|
|
20
39
|
checkGitIgnored: z.boolean().optional()
|
|
21
40
|
.describe('Whether to check if non-ignored files link to git-ignored files (default: true)'),
|
|
41
|
+
externalUrls: ExternalUrlValidationSchema.optional()
|
|
42
|
+
.describe('External URL validation configuration'),
|
|
22
43
|
}).describe('Validation configuration for a collection');
|
|
23
44
|
/**
|
|
24
45
|
* Configuration for a named collection of resources.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"project-config.js","sourceRoot":"","sources":["../../src/schemas/project-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;KACjE,QAAQ,CAAC,mDAAmD,CAAC,CAAC;AAIjE;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACrC,QAAQ,CAAC,mJAAmJ,CAAC;IAChK,IAAI,EAAE,oBAAoB,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,mCAAmC,CAAC;IAChD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,yDAAyD,CAAC;IACtE,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACpC,QAAQ,CAAC,iFAAiF,CAAC;
|
|
1
|
+
{"version":3,"file":"project-config.js","sourceRoot":"","sources":["../../src/schemas/project-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;KACjE,QAAQ,CAAC,mDAAmD,CAAC,CAAC;AAIjE;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAC5B,QAAQ,CAAC,oDAAoD,CAAC;IACjE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;SAC5C,QAAQ,CAAC,kDAAkD,CAAC;IAC/D,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAC/B,QAAQ,CAAC,sDAAsD,CAAC;IACnE,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SAC3C,QAAQ,CAAC,yEAAyE,CAAC;CACvF,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC,CAAC;AAIrD;;GAEG;AACH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SACrC,QAAQ,CAAC,mJAAmJ,CAAC;IAChK,IAAI,EAAE,oBAAoB,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,mCAAmC,CAAC;IAChD,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SAClC,QAAQ,CAAC,yDAAyD,CAAC;IACtE,eAAe,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;SACpC,QAAQ,CAAC,iFAAiF,CAAC;IAC9F,YAAY,EAAE,2BAA2B,CAAC,QAAQ,EAAE;SACjD,QAAQ,CAAC,uCAAuC,CAAC;CACrD,CAAC,CAAC,QAAQ,CAAC,2CAA2C,CAAC,CAAC;AAIzD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;SAChC,QAAQ,CAAC,qDAAqD,CAAC;IAClE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SACpC,QAAQ,CAAC,0BAA0B,CAAC;IACvC,UAAU,EAAE,0BAA0B,CAAC,QAAQ,EAAE;SAC9C,QAAQ,CAAC,8CAA8C,CAAC;CAC5D,CAAC,CAAC,QAAQ,CAAC,mDAAmD,CAAC,CAAC;AAIjE;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SACpC,QAAQ,CAAC,sEAAsE,CAAC;IACnF,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;SACpC,QAAQ,CAAC,sEAAsE,CAAC;IACnF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC;SACtD,QAAQ,CAAC,gCAAgC,CAAC;CAC9C,CAAC,CAAC,QAAQ,CAAC,4CAA4C,CAAC,CAAC;AAI1D;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;SAClB,QAAQ,CAAC,iCAAiC,CAAC;IAC9C,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;SACxC,QAAQ,CAAC,yBAAyB,CAAC;CACvC,CAAC,CAAC,QAAQ,CAAC,0CAA0C,CAAC,CAAC"}
|
|
@@ -8,6 +8,9 @@ import { z } from 'zod';
|
|
|
8
8
|
* - frontmatter_missing: Schema requires frontmatter, file has none
|
|
9
9
|
* - frontmatter_invalid_yaml: YAML syntax error in frontmatter
|
|
10
10
|
* - frontmatter_schema_error: Frontmatter fails JSON Schema validation
|
|
11
|
+
* - external_url_dead: External URL returned error status (4xx, 5xx)
|
|
12
|
+
* - external_url_timeout: External URL request timed out
|
|
13
|
+
* - external_url_error: External URL validation failed (DNS, network, etc.)
|
|
11
14
|
* - unknown_link: Unknown link type
|
|
12
15
|
*
|
|
13
16
|
* Includes details about what went wrong, where it occurred, and optionally
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validation-result.d.ts","sourceRoot":"","sources":["../../src/schemas/validation-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB
|
|
1
|
+
{"version":3,"file":"validation-result.d.ts","sourceRoot":"","sources":["../../src/schemas/validation-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;;;;;;GAgBG;AACH,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;;;;;;;;;EAOmC,CAAC;AAEtE,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EASsC,CAAC;AAE1E,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC"}
|
|
@@ -8,6 +8,9 @@ import { z } from 'zod';
|
|
|
8
8
|
* - frontmatter_missing: Schema requires frontmatter, file has none
|
|
9
9
|
* - frontmatter_invalid_yaml: YAML syntax error in frontmatter
|
|
10
10
|
* - frontmatter_schema_error: Frontmatter fails JSON Schema validation
|
|
11
|
+
* - external_url_dead: External URL returned error status (4xx, 5xx)
|
|
12
|
+
* - external_url_timeout: External URL request timed out
|
|
13
|
+
* - external_url_error: External URL validation failed (DNS, network, etc.)
|
|
11
14
|
* - unknown_link: Unknown link type
|
|
12
15
|
*
|
|
13
16
|
* Includes details about what went wrong, where it occurred, and optionally
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validation-result.js","sourceRoot":"","sources":["../../src/schemas/validation-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB
|
|
1
|
+
{"version":3,"file":"validation-result.js","sourceRoot":"","sources":["../../src/schemas/validation-result.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,oDAAoD,CAAC;IACvF,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,oCAAoC,CAAC;IAC3F,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0GAA0G,CAAC;IACrI,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;IACjD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,yCAAyC,CAAC;IACvE,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;CACvF,CAAC,CAAC,QAAQ,CAAC,wDAAwD,CAAC,CAAC;AAItE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC9F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,kDAAkD,CAAC;IACvG,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,kEAAkE,CAAC;IAC9I,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IAC9E,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAC7E,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;IAC/E,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACpF,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CAC9D,CAAC,CAAC,QAAQ,CAAC,4DAA4D,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vibe-agent-toolkit/resources",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.13",
|
|
4
4
|
"description": "Markdown resource parsing, validation, and link integrity checking",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -33,9 +33,10 @@
|
|
|
33
33
|
"author": "Jeff Dutton",
|
|
34
34
|
"license": "MIT",
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@vibe-agent-toolkit/utils": "0.1.
|
|
36
|
+
"@vibe-agent-toolkit/utils": "0.1.13",
|
|
37
37
|
"ajv": "^8.17.1",
|
|
38
38
|
"js-yaml": "^4.1.1",
|
|
39
|
+
"markdown-link-check": "^3.14.2",
|
|
39
40
|
"picomatch": "^4.0.3",
|
|
40
41
|
"remark-frontmatter": "^5.0.0",
|
|
41
42
|
"remark-gfm": "^4.0.0",
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { promises as fs } from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Cache entry for external link validation results
|
|
7
|
+
*/
|
|
8
|
+
interface CacheEntry {
|
|
9
|
+
statusCode: number;
|
|
10
|
+
statusMessage: string;
|
|
11
|
+
timestamp: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Cache storage format
|
|
16
|
+
*/
|
|
17
|
+
interface CacheData {
|
|
18
|
+
[url: string]: CacheEntry;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* External link validation cache
|
|
23
|
+
*
|
|
24
|
+
* Stores results of external URL checks to minimize redundant network requests.
|
|
25
|
+
* Uses file-based storage for persistence across runs.
|
|
26
|
+
*
|
|
27
|
+
* Cache keys are SHA-256 hashes of normalized URLs to handle long URLs and
|
|
28
|
+
* special characters safely in filenames.
|
|
29
|
+
*
|
|
30
|
+
* Example:
|
|
31
|
+
* ```typescript
|
|
32
|
+
* const cache = new ExternalLinkCache('/tmp/cache', 24);
|
|
33
|
+
*
|
|
34
|
+
* // Store a result
|
|
35
|
+
* await cache.set('https://example.com', 200, 'OK');
|
|
36
|
+
*
|
|
37
|
+
* // Retrieve a result
|
|
38
|
+
* const result = await cache.get('https://example.com');
|
|
39
|
+
* if (result) {
|
|
40
|
+
* console.log(`Status: ${result.statusCode}`);
|
|
41
|
+
* }
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
export class ExternalLinkCache {
|
|
45
|
+
private readonly cacheDir: string;
|
|
46
|
+
private readonly ttlHours: number;
|
|
47
|
+
private readonly cacheFile: string;
|
|
48
|
+
private cache: CacheData | null = null;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Create a new external link cache
|
|
52
|
+
*
|
|
53
|
+
* @param cacheDir - Directory to store cache files
|
|
54
|
+
* @param ttlHours - Time-to-live in hours (default: 24)
|
|
55
|
+
*/
|
|
56
|
+
constructor(cacheDir: string, ttlHours = 24) {
|
|
57
|
+
this.cacheDir = cacheDir;
|
|
58
|
+
this.ttlHours = ttlHours;
|
|
59
|
+
this.cacheFile = path.join(cacheDir, 'external-links.json');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Load cache from disk
|
|
64
|
+
*/
|
|
65
|
+
private async loadCache(): Promise<CacheData> {
|
|
66
|
+
if (this.cache !== null) {
|
|
67
|
+
return this.cache;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
try {
|
|
71
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- cacheDir is constructor parameter, controlled by caller
|
|
72
|
+
await fs.mkdir(this.cacheDir, { recursive: true });
|
|
73
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- cacheFile is derived from cacheDir
|
|
74
|
+
const data = await fs.readFile(this.cacheFile, 'utf-8');
|
|
75
|
+
this.cache = JSON.parse(data) as CacheData;
|
|
76
|
+
return this.cache;
|
|
77
|
+
} catch (error) {
|
|
78
|
+
// Handle missing file or corrupted JSON - start with empty cache
|
|
79
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT' || error instanceof SyntaxError) {
|
|
80
|
+
this.cache = {};
|
|
81
|
+
return this.cache;
|
|
82
|
+
}
|
|
83
|
+
throw error;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Save cache to disk
|
|
89
|
+
*/
|
|
90
|
+
private async saveCache(): Promise<void> {
|
|
91
|
+
if (this.cache === null) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- cacheDir is constructor parameter, controlled by caller
|
|
96
|
+
await fs.mkdir(this.cacheDir, { recursive: true });
|
|
97
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename -- cacheFile is derived from cacheDir
|
|
98
|
+
await fs.writeFile(this.cacheFile, JSON.stringify(this.cache, null, 2), 'utf-8');
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Normalize URL for caching
|
|
103
|
+
*
|
|
104
|
+
* Removes trailing slashes and anchors to treat variations as the same URL.
|
|
105
|
+
*/
|
|
106
|
+
private normalizeUrl(url: string): string {
|
|
107
|
+
try {
|
|
108
|
+
const parsed = new URL(url);
|
|
109
|
+
// Remove hash/anchor
|
|
110
|
+
parsed.hash = '';
|
|
111
|
+
// Remove trailing slash
|
|
112
|
+
let normalized = parsed.toString();
|
|
113
|
+
if (normalized.endsWith('/')) {
|
|
114
|
+
normalized = normalized.slice(0, -1);
|
|
115
|
+
}
|
|
116
|
+
return normalized;
|
|
117
|
+
} catch {
|
|
118
|
+
// If URL parsing fails, use as-is
|
|
119
|
+
return url;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Generate cache key for URL
|
|
125
|
+
*
|
|
126
|
+
* Uses SHA-256 hash to handle long URLs and special characters.
|
|
127
|
+
*/
|
|
128
|
+
private getCacheKey(url: string): string {
|
|
129
|
+
const normalized = this.normalizeUrl(url);
|
|
130
|
+
return createHash('sha256').update(normalized).digest('hex');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Check if cache entry is expired
|
|
135
|
+
*/
|
|
136
|
+
private isExpired(entry: CacheEntry): boolean {
|
|
137
|
+
const now = Date.now();
|
|
138
|
+
const age = now - entry.timestamp;
|
|
139
|
+
const ttlMs = this.ttlHours * 60 * 60 * 1000;
|
|
140
|
+
return age > ttlMs;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Get cached result for URL
|
|
145
|
+
*
|
|
146
|
+
* @param url - URL to look up
|
|
147
|
+
* @returns Cache entry or null if not found/expired
|
|
148
|
+
*/
|
|
149
|
+
async get(url: string): Promise<CacheEntry | null> {
|
|
150
|
+
const cache = await this.loadCache();
|
|
151
|
+
const key = this.getCacheKey(url);
|
|
152
|
+
const entry = cache[key];
|
|
153
|
+
|
|
154
|
+
if (!entry) {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (this.isExpired(entry)) {
|
|
159
|
+
delete cache[key];
|
|
160
|
+
await this.saveCache();
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return entry;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Store validation result in cache
|
|
169
|
+
*
|
|
170
|
+
* @param url - URL to cache
|
|
171
|
+
* @param statusCode - HTTP status code
|
|
172
|
+
* @param statusMessage - HTTP status message
|
|
173
|
+
*/
|
|
174
|
+
async set(url: string, statusCode: number, statusMessage: string): Promise<void> {
|
|
175
|
+
const cache = await this.loadCache();
|
|
176
|
+
const key = this.getCacheKey(url);
|
|
177
|
+
|
|
178
|
+
cache[key] = {
|
|
179
|
+
statusCode,
|
|
180
|
+
statusMessage,
|
|
181
|
+
timestamp: Date.now(),
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
await this.saveCache();
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Clear all cache entries
|
|
189
|
+
*/
|
|
190
|
+
async clear(): Promise<void> {
|
|
191
|
+
this.cache = {};
|
|
192
|
+
await this.saveCache();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Get cache statistics
|
|
197
|
+
*/
|
|
198
|
+
async getStats(): Promise<{ total: number; expired: number }> {
|
|
199
|
+
const cache = await this.loadCache();
|
|
200
|
+
const keys = Object.keys(cache);
|
|
201
|
+
const expired = keys.filter((key) => {
|
|
202
|
+
const entry = cache[key];
|
|
203
|
+
// Type guard: ensure entry exists before checking expiration
|
|
204
|
+
if (entry === undefined) {
|
|
205
|
+
return false;
|
|
206
|
+
}
|
|
207
|
+
return this.isExpired(entry);
|
|
208
|
+
}).length;
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
total: keys.length,
|
|
212
|
+
expired,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
import markdownLinkCheck from 'markdown-link-check';
|
|
2
|
+
|
|
3
|
+
import { ExternalLinkCache } from './external-link-cache.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Safely serialize an error to a string, preventing [object Object] issues.
|
|
7
|
+
* Handles Error objects, strings, objects, and edge cases.
|
|
8
|
+
*/
|
|
9
|
+
function safeSerializeError(err: unknown): string | undefined {
|
|
10
|
+
if (!err) {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
if (typeof err === 'string') {
|
|
15
|
+
// Return undefined for empty strings so fallback message is used
|
|
16
|
+
return err.trim() || undefined;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (err instanceof Error) {
|
|
20
|
+
// Return undefined for empty messages so fallback is used
|
|
21
|
+
return err.message.trim() || undefined;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// For objects, try JSON.stringify with fallback
|
|
25
|
+
try {
|
|
26
|
+
const serialized = JSON.stringify(err);
|
|
27
|
+
// Avoid returning literal "{}" which isn't helpful
|
|
28
|
+
if (serialized === '{}') {
|
|
29
|
+
// Try to extract something useful from the object
|
|
30
|
+
const msg = (err as { message?: unknown }).message;
|
|
31
|
+
return typeof msg === 'string' && msg.trim() ? msg : 'Unknown error';
|
|
32
|
+
}
|
|
33
|
+
return serialized;
|
|
34
|
+
} catch {
|
|
35
|
+
// JSON.stringify can fail on circular references
|
|
36
|
+
// Try to extract message property if it exists
|
|
37
|
+
const msg = (err as { message?: unknown }).message;
|
|
38
|
+
return typeof msg === 'string' && msg.trim() ? msg : 'Error (unserializable)';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Configuration options for external link validation
|
|
44
|
+
*/
|
|
45
|
+
export interface ExternalLinkValidatorOptions {
|
|
46
|
+
/** Time-to-live for cache entries in hours (default: 24) */
|
|
47
|
+
cacheTtlHours?: number;
|
|
48
|
+
/** Request timeout in milliseconds (default: 5000) */
|
|
49
|
+
timeout?: number;
|
|
50
|
+
/** Number of retries for failed requests (default: 2) */
|
|
51
|
+
retries?: number;
|
|
52
|
+
/** User agent string for requests (default: generic) */
|
|
53
|
+
userAgent?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Result of validating a single external link
|
|
58
|
+
*/
|
|
59
|
+
export interface LinkValidationResult {
|
|
60
|
+
/** The URL that was validated */
|
|
61
|
+
url: string;
|
|
62
|
+
/** Validation status: 'ok' = working, 'error' = broken */
|
|
63
|
+
status: 'ok' | 'error';
|
|
64
|
+
/** HTTP status code (e.g., 200, 404) */
|
|
65
|
+
statusCode: number;
|
|
66
|
+
/** Error message if validation failed */
|
|
67
|
+
error?: string;
|
|
68
|
+
/** Whether result came from cache */
|
|
69
|
+
cached: boolean;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Validates external URLs in markdown content
|
|
74
|
+
*
|
|
75
|
+
* Uses markdown-link-check library with caching to efficiently validate
|
|
76
|
+
* external links. Respects cache TTL and provides detailed error information.
|
|
77
|
+
*
|
|
78
|
+
* Example:
|
|
79
|
+
* ```typescript
|
|
80
|
+
* const validator = new ExternalLinkValidator('/tmp/cache', {
|
|
81
|
+
* cacheTtlHours: 24,
|
|
82
|
+
* timeout: 5000,
|
|
83
|
+
* });
|
|
84
|
+
*
|
|
85
|
+
* const result = await validator.validateLink('https://example.com');
|
|
86
|
+
* if (result.status === 'error') {
|
|
87
|
+
* console.error(`Broken link: ${result.url} - ${result.error}`);
|
|
88
|
+
* }
|
|
89
|
+
* ```
|
|
90
|
+
*/
|
|
91
|
+
export class ExternalLinkValidator {
|
|
92
|
+
private readonly cache: ExternalLinkCache;
|
|
93
|
+
private readonly options: Required<ExternalLinkValidatorOptions>;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Create a new external link validator
|
|
97
|
+
*
|
|
98
|
+
* @param cacheDir - Directory for storing cache
|
|
99
|
+
* @param options - Validation options
|
|
100
|
+
*/
|
|
101
|
+
constructor(cacheDir: string, options: ExternalLinkValidatorOptions = {}) {
|
|
102
|
+
this.options = {
|
|
103
|
+
cacheTtlHours: options.cacheTtlHours ?? 24,
|
|
104
|
+
timeout: options.timeout ?? 5000,
|
|
105
|
+
retries: options.retries ?? 2,
|
|
106
|
+
userAgent:
|
|
107
|
+
options.userAgent ??
|
|
108
|
+
'Mozilla/5.0 (compatible; VAT-LinkChecker/1.0; +https://github.com/jdutton/vibe-agent-toolkit)',
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
this.cache = new ExternalLinkCache(cacheDir, this.options.cacheTtlHours);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Validate a single external link
|
|
116
|
+
*
|
|
117
|
+
* @param url - URL to validate
|
|
118
|
+
* @returns Validation result
|
|
119
|
+
*/
|
|
120
|
+
async validateLink(url: string): Promise<LinkValidationResult> {
|
|
121
|
+
// Check cache first
|
|
122
|
+
const cached = await this.cache.get(url);
|
|
123
|
+
if (cached) {
|
|
124
|
+
const isOk = cached.statusCode >= 200 && cached.statusCode < 400;
|
|
125
|
+
|
|
126
|
+
// Return success result without error property (exactOptionalPropertyTypes)
|
|
127
|
+
if (isOk) {
|
|
128
|
+
return {
|
|
129
|
+
url,
|
|
130
|
+
status: 'ok' as const,
|
|
131
|
+
statusCode: cached.statusCode,
|
|
132
|
+
cached: true,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Return error result with error property
|
|
137
|
+
return {
|
|
138
|
+
url,
|
|
139
|
+
status: 'error' as const,
|
|
140
|
+
statusCode: cached.statusCode,
|
|
141
|
+
cached: true,
|
|
142
|
+
error: cached.statusMessage,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// Validate using markdown-link-check
|
|
147
|
+
const result = await this.checkLink(url);
|
|
148
|
+
|
|
149
|
+
// Store in cache
|
|
150
|
+
await this.cache.set(url, result.statusCode, result.error ?? 'OK');
|
|
151
|
+
|
|
152
|
+
return {
|
|
153
|
+
...result,
|
|
154
|
+
cached: false,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Validate multiple links
|
|
160
|
+
*
|
|
161
|
+
* @param urls - URLs to validate
|
|
162
|
+
* @returns Array of validation results
|
|
163
|
+
*/
|
|
164
|
+
async validateLinks(urls: string[]): Promise<LinkValidationResult[]> {
|
|
165
|
+
return Promise.all(urls.map((url) => this.validateLink(url)));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Check a link using markdown-link-check
|
|
170
|
+
*/
|
|
171
|
+
private async checkLink(
|
|
172
|
+
url: string,
|
|
173
|
+
): Promise<Pick<LinkValidationResult, 'url' | 'status' | 'statusCode' | 'error'>> {
|
|
174
|
+
return new Promise((resolve) => {
|
|
175
|
+
const markdown = `[link](${url})`;
|
|
176
|
+
|
|
177
|
+
markdownLinkCheck(
|
|
178
|
+
markdown,
|
|
179
|
+
{
|
|
180
|
+
timeout: `${this.options.timeout}ms`,
|
|
181
|
+
retryOn429: true,
|
|
182
|
+
retryCount: this.options.retries,
|
|
183
|
+
aliveStatusCodes: [200, 206, 301, 302, 307, 308],
|
|
184
|
+
ignorePatterns: [],
|
|
185
|
+
httpHeaders: [
|
|
186
|
+
{
|
|
187
|
+
urls: [url],
|
|
188
|
+
headers: {
|
|
189
|
+
'User-Agent': this.options.userAgent,
|
|
190
|
+
},
|
|
191
|
+
},
|
|
192
|
+
],
|
|
193
|
+
},
|
|
194
|
+
(error: Error | null, results: Array<{ link: string; status: string; statusCode: number; err?: string | Error | object }>) => {
|
|
195
|
+
if (error) {
|
|
196
|
+
resolve({
|
|
197
|
+
url,
|
|
198
|
+
status: 'error',
|
|
199
|
+
statusCode: 0,
|
|
200
|
+
error: error.message,
|
|
201
|
+
});
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const result = results[0];
|
|
206
|
+
if (!result) {
|
|
207
|
+
resolve({
|
|
208
|
+
url,
|
|
209
|
+
status: 'error',
|
|
210
|
+
statusCode: 0,
|
|
211
|
+
error: 'No result from markdown-link-check',
|
|
212
|
+
});
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if (result.status === 'alive') {
|
|
217
|
+
resolve({
|
|
218
|
+
url,
|
|
219
|
+
status: 'ok',
|
|
220
|
+
statusCode: result.statusCode,
|
|
221
|
+
});
|
|
222
|
+
} else {
|
|
223
|
+
const errorMessage = safeSerializeError(result.err) ?? `Link status: ${result.status}`;
|
|
224
|
+
|
|
225
|
+
resolve({
|
|
226
|
+
url,
|
|
227
|
+
status: 'error',
|
|
228
|
+
statusCode: result.statusCode,
|
|
229
|
+
error: errorMessage,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
},
|
|
233
|
+
);
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Clear the validation cache
|
|
239
|
+
*/
|
|
240
|
+
async clearCache(): Promise<void> {
|
|
241
|
+
await this.cache.clear();
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Get cache statistics
|
|
246
|
+
*/
|
|
247
|
+
async getCacheStats(): Promise<{ total: number; expired: number }> {
|
|
248
|
+
return this.cache.getStats();
|
|
249
|
+
}
|
|
250
|
+
}
|
package/src/link-parser.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
import { readFile, stat } from 'node:fs/promises';
|
|
13
13
|
|
|
14
14
|
import * as yaml from 'js-yaml';
|
|
15
|
-
import type { Heading, Link, LinkReference, Root } from 'mdast';
|
|
15
|
+
import type { Definition, Heading, Link, LinkReference, Root } from 'mdast';
|
|
16
16
|
import remarkFrontmatter from 'remark-frontmatter';
|
|
17
17
|
import remarkGfm from 'remark-gfm';
|
|
18
18
|
import remarkParse from 'remark-parse';
|
|
@@ -130,6 +130,18 @@ function extractLinks(tree: Root): ResourceLink[] {
|
|
|
130
130
|
links.push(link);
|
|
131
131
|
});
|
|
132
132
|
|
|
133
|
+
// Visit definition nodes (reference-style link definitions: [ref]: url)
|
|
134
|
+
// These provide the actual URLs for linkReference nodes
|
|
135
|
+
visit(tree, 'definition', (node: Definition) => {
|
|
136
|
+
const link: ResourceLink = {
|
|
137
|
+
text: node.identifier,
|
|
138
|
+
href: node.url,
|
|
139
|
+
type: classifyLink(node.url),
|
|
140
|
+
line: node.position?.start.line,
|
|
141
|
+
};
|
|
142
|
+
links.push(link);
|
|
143
|
+
});
|
|
144
|
+
|
|
133
145
|
return links;
|
|
134
146
|
}
|
|
135
147
|
|