cook-ssg 2.1.2 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -20
- package/package.json +4 -7
- package/scripts/build.js +98 -26
- package/scripts/dev.js +46 -16
- package/scripts/plugins/apply-template.js +8 -17
- package/scripts/plugins/auto-components.js +2 -2
- package/scripts/plugins/bundle.js +6 -3
- package/scripts/plugins/copy-src.js +1 -1
- package/scripts/plugins/create-dir-from-file.js +4 -2
- package/scripts/plugins/custom-plugins.js +1 -1
- package/scripts/plugins/declarative-shadow-dom.js +2 -2
- package/scripts/plugins/escape-code-blocks.js +22 -1
- package/scripts/plugins/fingerprint-assets.js +388 -0
- package/scripts/plugins/generate-formats.js +49 -39
- package/scripts/plugins/generate-fragments.js +24 -14
- package/scripts/plugins/generate-sitemap-xml.js +38 -15
- package/scripts/plugins/minify-src.js +98 -72
- package/scripts/plugins/optimize-images.js +80 -26
- package/scripts/plugins/process-markdown.js +12 -25
- package/scripts/plugins/repeat-collection.js +70 -45
- package/scripts/plugins/replace-components.js +82 -60
- package/scripts/plugins/replace-external-link-protocol.js +16 -31
- package/scripts/plugins/replace-include.js +34 -19
- package/scripts/plugins/replace-inline.js +6 -3
- package/scripts/plugins/replace-template-strings.js +2 -50
- package/scripts/plugins/set-active-links.js +57 -26
- package/scripts/utils/config/config.js +44 -5
- package/scripts/utils/get-src/get-src.js +9 -45
- package/scripts/utils/interpolate/interpolate.js +83 -0
- package/scripts/utils/logger/logger.js +8 -3
- package/scripts/utils/site-pages/site-pages.js +37 -0
- package/scripts/utils/source-file/source-file.js +64 -0
- package/scripts/utils/user-data/user-data.js +45 -0
- package/scripts/utils/util/util.js +88 -46
- package/patches/jsdom-fetch-fix.js +0 -52
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file fingerprint-assets.js
|
|
3
|
+
* @description Content-hash fingerprinting for built CSS and JS assets.
|
|
4
|
+
*
|
|
5
|
+
* Runs as a post-pass over dist/ after all HTML and bundles are written.
|
|
6
|
+
*
|
|
7
|
+
* Phase 1: Hash and rename matching assets (foo.css -> foo.a1b2c3d4.css).
|
|
8
|
+
* Source map files (.map) are renamed alongside their source and the
|
|
9
|
+
* sourceMappingURL comment is updated to point to the new map name.
|
|
10
|
+
* Phase 2: Walk dist/ HTML and rewrite <link href> and <script src> refs.
|
|
11
|
+
* Phase 3: Emit dist/<manifestPath> mapping original -> hashed paths.
|
|
12
|
+
*
|
|
13
|
+
* Out of scope for v1: url() refs inside CSS, image fingerprinting.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
// IMPORTS
|
|
17
|
+
// -----------------------------
|
|
18
|
+
import { createHash } from 'node:crypto';
|
|
19
|
+
import { promises as fs } from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
import Logger from '../utils/logger/logger.js';
|
|
22
|
+
import Util from '../utils/util/util.js';
|
|
23
|
+
|
|
24
|
+
// Config
|
|
25
|
+
import { distPath, fingerprint as fingerprintConfig } from '../utils/config/config.js';
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
// ============================================================================
|
|
29
|
+
// Pure helpers (extracted for unit testing)
|
|
30
|
+
// ============================================================================
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Compute a content hash truncated to the given length.
|
|
34
|
+
* @param {Buffer|string} content
|
|
35
|
+
* @param {number} length - Number of hex chars to keep
|
|
36
|
+
* @param {string} [algorithm='sha256']
|
|
37
|
+
* @returns {string}
|
|
38
|
+
*/
|
|
39
|
+
export function computeHash(content, length, algorithm = 'sha256') {
|
|
40
|
+
return createHash(algorithm).update(content).digest('hex').slice(0, length);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Insert a hash into a filename before the extension.
|
|
45
|
+
* Handles compound extensions like `.js.map` by inserting before the last
|
|
46
|
+
* meaningful extension (e.g. `foo.js.map` -> `foo.{hash}.js.map`).
|
|
47
|
+
* @param {string} filename - Base filename, e.g. "foo.css"
|
|
48
|
+
* @param {string} hash
|
|
49
|
+
* @returns {string} e.g. "foo.{hash}.css"
|
|
50
|
+
*/
|
|
51
|
+
export function fingerprintFilename(filename, hash) {
|
|
52
|
+
// Map files: keep .js.map / .css.map paired with their source hash
|
|
53
|
+
if (filename.endsWith('.js.map')) {
|
|
54
|
+
const stem = filename.slice(0, -'.js.map'.length);
|
|
55
|
+
return `${stem}.${hash}.js.map`;
|
|
56
|
+
}
|
|
57
|
+
if (filename.endsWith('.css.map')) {
|
|
58
|
+
const stem = filename.slice(0, -'.css.map'.length);
|
|
59
|
+
return `${stem}.${hash}.css.map`;
|
|
60
|
+
}
|
|
61
|
+
const ext = path.extname(filename);
|
|
62
|
+
if (!ext) return `${filename}.${hash}`;
|
|
63
|
+
const stem = filename.slice(0, -ext.length);
|
|
64
|
+
return `${stem}.${hash}${ext}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Should this href/src be left alone?
|
|
69
|
+
* @param {string} url
|
|
70
|
+
* @returns {boolean}
|
|
71
|
+
*/
|
|
72
|
+
export function isExternalRef(url) {
|
|
73
|
+
if (!url) return true;
|
|
74
|
+
if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) return true;
|
|
75
|
+
if (url.startsWith('data:')) return true;
|
|
76
|
+
if (url.startsWith('mailto:') || url.startsWith('tel:')) return true;
|
|
77
|
+
return false;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Strip query string / fragment from a URL for manifest lookup.
|
|
82
|
+
* Returns { path, suffix } so the suffix can be reattached after rewriting.
|
|
83
|
+
* @param {string} url
|
|
84
|
+
*/
|
|
85
|
+
export function splitUrlSuffix(url) {
|
|
86
|
+
const qIdx = url.search(/[?#]/);
|
|
87
|
+
if (qIdx === -1) return { path: url, suffix: '' };
|
|
88
|
+
return { path: url.slice(0, qIdx), suffix: url.slice(qIdx) };
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Rewrite <link href> and <script src> in an HTML string using the manifest.
|
|
93
|
+
* Manifest keys are absolute-from-dist paths (e.g. "/assets/css/site.css").
|
|
94
|
+
* @param {string} html
|
|
95
|
+
* @param {Object} manifest - { "/assets/css/site.css": "/assets/css/site.{hash}.css", ... }
|
|
96
|
+
* @returns {{ html: string, rewrites: number }}
|
|
97
|
+
*/
|
|
98
|
+
export function rewriteHrefs(html, manifest) {
|
|
99
|
+
if (!manifest || Object.keys(manifest).length === 0) return { html, rewrites: 0 };
|
|
100
|
+
|
|
101
|
+
// linkedom round-trips exactly what was parsed: fragments stay
|
|
102
|
+
// fragments (real JSDOM wrapped them in <html><head><body>, corrupting
|
|
103
|
+
// _fragment.html files consumed by SPA navigation).
|
|
104
|
+
const dom = Util.jsdom.dom({ src: html });
|
|
105
|
+
const document = dom.window.document;
|
|
106
|
+
let rewrites = 0;
|
|
107
|
+
|
|
108
|
+
const rewriteAttr = (el, attr) => {
|
|
109
|
+
const original = el.getAttribute(attr);
|
|
110
|
+
if (!original || isExternalRef(original)) return;
|
|
111
|
+
const { path: cleanPath, suffix } = splitUrlSuffix(original);
|
|
112
|
+
const lookupKey = cleanPath.startsWith('/') ? cleanPath : `/${cleanPath}`;
|
|
113
|
+
const replacement = manifest[lookupKey];
|
|
114
|
+
if (!replacement) return;
|
|
115
|
+
// Preserve leading-slash style of the original
|
|
116
|
+
const newValue = cleanPath.startsWith('/') ? replacement : replacement.replace(/^\//, '');
|
|
117
|
+
el.setAttribute(attr, `${newValue}${suffix}`);
|
|
118
|
+
rewrites += 1;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
document.querySelectorAll('link[href]').forEach(el => rewriteAttr(el, 'href'));
|
|
122
|
+
document.querySelectorAll('script[src]').forEach(el => rewriteAttr(el, 'src'));
|
|
123
|
+
|
|
124
|
+
if (rewrites === 0) return { html, rewrites: 0 };
|
|
125
|
+
return { html: Util.setSrc({ dom }), rewrites };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Matches ES module specifiers: `import x from '...'`, `export ... from '...'`,
|
|
130
|
+
* side-effect `import '...'`, and dynamic `import('...')`.
|
|
131
|
+
*/
|
|
132
|
+
const SPECIFIER_PATTERN = /((?:\bfrom|\bimport)\s*\(?\s*['"])([^'"]+)(['"])/g;
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
// ============================================================================
|
|
136
|
+
// Plugin
|
|
137
|
+
// ============================================================================
|
|
138
|
+
|
|
139
|
+
class FingerprintAssets {
|
|
140
|
+
constructor(opts = {}) {
|
|
141
|
+
// Optional per-call overrides (used by tests and custom pipelines)
|
|
142
|
+
this.distPath = opts.distPath || distPath;
|
|
143
|
+
this.config = {
|
|
144
|
+
enabled: false,
|
|
145
|
+
hashLength: 8,
|
|
146
|
+
algorithm: 'sha256',
|
|
147
|
+
types: ['.css', '.js'],
|
|
148
|
+
excludePaths: [],
|
|
149
|
+
manifestPath: 'asset-manifest.json',
|
|
150
|
+
...fingerprintConfig,
|
|
151
|
+
...(opts.fingerprint || {}),
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
async init() {
|
|
156
|
+
if (!this.config.enabled) return;
|
|
157
|
+
if (process.env.DEV_CHANGED_PAGE) return;
|
|
158
|
+
|
|
159
|
+
Logger.persist.header('\nFingerprint Assets');
|
|
160
|
+
|
|
161
|
+
// Phase 1: Find candidates and hash + rename
|
|
162
|
+
const candidates = await this.findCandidates();
|
|
163
|
+
if (!candidates.length) {
|
|
164
|
+
Logger.persist.success('No matching assets (skipped)');
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const manifest = await this.fingerprintFiles(candidates);
|
|
169
|
+
|
|
170
|
+
// Phase 2: Rewrite HTML references
|
|
171
|
+
const htmlFiles = (Util.getPaths(this.distPath, this.distPath, []) || [])
|
|
172
|
+
.filter(p => p.endsWith('.html'));
|
|
173
|
+
let totalRewrites = 0;
|
|
174
|
+
for (const htmlPath of htmlFiles) {
|
|
175
|
+
const original = await fs.readFile(htmlPath, 'utf-8');
|
|
176
|
+
const { html, rewrites } = rewriteHrefs(original, manifest);
|
|
177
|
+
if (rewrites > 0) {
|
|
178
|
+
await fs.writeFile(htmlPath, html, 'utf-8');
|
|
179
|
+
totalRewrites += rewrites;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Phase 3: Write manifest
|
|
184
|
+
const manifestFsPath = path.join(this.distPath, this.config.manifestPath);
|
|
185
|
+
await fs.writeFile(manifestFsPath, JSON.stringify(manifest, null, 2), 'utf-8');
|
|
186
|
+
|
|
187
|
+
Logger.persist.success(
|
|
188
|
+
`Fingerprinted ${candidates.length} asset(s), rewrote ${totalRewrites} HTML ref(s)`
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Find all dist files matching configured types, excluding configured paths.
|
|
194
|
+
* Returns absolute paths.
|
|
195
|
+
*/
|
|
196
|
+
async findCandidates() {
|
|
197
|
+
const all = Util.getPaths(this.distPath, this.distPath, []) || [];
|
|
198
|
+
const excludes = this.config.excludePaths.map(p => (p instanceof RegExp ? p : new RegExp(p)));
|
|
199
|
+
return all.filter(p => {
|
|
200
|
+
// Skip source maps in the candidate list — handled alongside their source.
|
|
201
|
+
if (p.endsWith('.map')) return false;
|
|
202
|
+
const ext = path.extname(p);
|
|
203
|
+
if (!this.config.types.includes(ext)) return false;
|
|
204
|
+
if (excludes.some(r => r.test(p))) return false;
|
|
205
|
+
return true;
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Hash, rename, and (if present) rename the paired .map and patch sourceMappingURL.
|
|
211
|
+
* @param {string[]} candidates - Absolute paths
|
|
212
|
+
* @returns {Promise<Object>} manifest of relative-from-dist mappings
|
|
213
|
+
*/
|
|
214
|
+
async fingerprintFiles(candidates) {
|
|
215
|
+
const manifest = {};
|
|
216
|
+
|
|
217
|
+
// Read all candidates up front so we can order JS files by their
|
|
218
|
+
// import graph: a dependency must be hashed before its importers,
|
|
219
|
+
// because the importer's content embeds the dependency's hashed name.
|
|
220
|
+
// Otherwise an importer keeps its old hash when a dependency changes,
|
|
221
|
+
// and immutable caching serves a stale importer pointing at a dead file.
|
|
222
|
+
const contents = new Map();
|
|
223
|
+
for (const p of candidates) contents.set(p, await fs.readFile(p));
|
|
224
|
+
const ordered = this.orderByDependencies(candidates, contents);
|
|
225
|
+
|
|
226
|
+
for (const absPath of ordered) {
|
|
227
|
+
let buffer = contents.get(absPath);
|
|
228
|
+
let contentChanged = false;
|
|
229
|
+
|
|
230
|
+
// Rewrite import specifiers of already-fingerprinted dependencies
|
|
231
|
+
// BEFORE hashing, so this file's hash reflects theirs
|
|
232
|
+
if (absPath.endsWith('.js')) {
|
|
233
|
+
const { code, changed } = this.rewriteImportSpecifiers(
|
|
234
|
+
buffer.toString('utf-8'), absPath, manifest
|
|
235
|
+
);
|
|
236
|
+
if (changed) {
|
|
237
|
+
buffer = Buffer.from(code, 'utf-8');
|
|
238
|
+
contentChanged = true;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
const hash = computeHash(buffer, this.config.hashLength, this.config.algorithm);
|
|
243
|
+
|
|
244
|
+
const dir = path.dirname(absPath);
|
|
245
|
+
const baseName = path.basename(absPath);
|
|
246
|
+
const newName = fingerprintFilename(baseName, hash);
|
|
247
|
+
const newAbs = path.join(dir, newName);
|
|
248
|
+
|
|
249
|
+
// Handle paired .map file (if present)
|
|
250
|
+
const mapAbs = `${absPath}.map`;
|
|
251
|
+
let updatedContent = null;
|
|
252
|
+
let mapNewName = null;
|
|
253
|
+
try {
|
|
254
|
+
await fs.access(mapAbs);
|
|
255
|
+
// Rename map file with the same hash
|
|
256
|
+
mapNewName = fingerprintFilename(`${baseName}.map`, hash);
|
|
257
|
+
const mapNewAbs = path.join(dir, mapNewName);
|
|
258
|
+
await fs.rename(mapAbs, mapNewAbs);
|
|
259
|
+
// Patch sourceMappingURL comment to point at the new map name
|
|
260
|
+
const text = buffer.toString('utf-8');
|
|
261
|
+
contentChanged = false; // superseded by updatedContent below
|
|
262
|
+
updatedContent = text.replace(
|
|
263
|
+
/(sourceMappingURL=)([^\s*'"]+)/,
|
|
264
|
+
(_, prefix) => `${prefix}${mapNewName}`
|
|
265
|
+
);
|
|
266
|
+
} catch {
|
|
267
|
+
// No map file
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
if (updatedContent !== null) {
|
|
271
|
+
await fs.writeFile(newAbs, updatedContent, 'utf-8');
|
|
272
|
+
await fs.unlink(absPath);
|
|
273
|
+
} else if (contentChanged) {
|
|
274
|
+
// Import specifiers were rewritten — write the updated content
|
|
275
|
+
await fs.writeFile(newAbs, buffer);
|
|
276
|
+
await fs.unlink(absPath);
|
|
277
|
+
} else {
|
|
278
|
+
await fs.rename(absPath, newAbs);
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Build manifest keys as web paths starting with `/`
|
|
282
|
+
const relOriginal = '/' + path.relative(this.distPath, absPath).split(path.sep).join('/');
|
|
283
|
+
const relNew = '/' + path.relative(this.distPath, newAbs).split(path.sep).join('/');
|
|
284
|
+
manifest[relOriginal] = relNew;
|
|
285
|
+
|
|
286
|
+
if (mapNewName) {
|
|
287
|
+
const relMapOriginal = '/' + path.relative(this.distPath, mapAbs).split(path.sep).join('/');
|
|
288
|
+
const relMapNew = '/' + path.relative(this.distPath, path.join(dir, mapNewName)).split(path.sep).join('/');
|
|
289
|
+
manifest[relMapOriginal] = relMapNew;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
return manifest;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Resolve an import specifier to a manifest key (absolute-from-dist web
|
|
298
|
+
* path), or null when it can't be fingerprinted (external URL, bare
|
|
299
|
+
* package specifier).
|
|
300
|
+
* @param {string} spec - Raw specifier from the source
|
|
301
|
+
* @param {string} absPath - File the specifier appears in
|
|
302
|
+
* @returns {string|null}
|
|
303
|
+
*/
|
|
304
|
+
resolveSpecifierKey(spec, absPath) {
|
|
305
|
+
if (isExternalRef(spec)) return null;
|
|
306
|
+
const clean = splitUrlSuffix(spec).path;
|
|
307
|
+
if (clean.startsWith('/')) return clean;
|
|
308
|
+
if (clean.startsWith('./') || clean.startsWith('../')) {
|
|
309
|
+
const dirRel = path.dirname(path.relative(this.distPath, absPath));
|
|
310
|
+
const joined = path.normalize(path.join(dirRel, clean));
|
|
311
|
+
return '/' + joined.split(path.sep).join('/');
|
|
312
|
+
}
|
|
313
|
+
// Bare specifier (npm package / import map) — leave alone
|
|
314
|
+
return null;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Rewrite import/export/dynamic-import specifiers whose targets exist in
|
|
319
|
+
* the manifest. Fingerprinting only changes the basename, so the
|
|
320
|
+
* specifier's directory part is preserved as written.
|
|
321
|
+
* @param {string} code
|
|
322
|
+
* @param {string} absPath - File the code came from
|
|
323
|
+
* @param {Object} manifest
|
|
324
|
+
* @returns {{ code: string, changed: boolean }}
|
|
325
|
+
*/
|
|
326
|
+
rewriteImportSpecifiers(code, absPath, manifest) {
|
|
327
|
+
let changed = false;
|
|
328
|
+
const out = code.replace(SPECIFIER_PATTERN, (match, pre, spec, post) => {
|
|
329
|
+
const key = this.resolveSpecifierKey(spec, absPath);
|
|
330
|
+
if (!key || !manifest[key]) return match;
|
|
331
|
+
const newBase = path.basename(manifest[key]);
|
|
332
|
+
const slashIdx = spec.lastIndexOf('/');
|
|
333
|
+
const newSpec = slashIdx === -1 ? newBase : spec.slice(0, slashIdx + 1) + newBase;
|
|
334
|
+
changed = true;
|
|
335
|
+
return `${pre}${newSpec}${post}`;
|
|
336
|
+
});
|
|
337
|
+
return { code: out, changed };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Order candidates so JS dependencies precede their importers
|
|
342
|
+
* (depth-first topological sort, cycle-tolerant).
|
|
343
|
+
* @param {string[]} candidates - Absolute paths
|
|
344
|
+
* @param {Map<string,Buffer>} contents
|
|
345
|
+
* @returns {string[]}
|
|
346
|
+
*/
|
|
347
|
+
orderByDependencies(candidates, contents) {
|
|
348
|
+
const keyOf = p => '/' + path.relative(this.distPath, p).split(path.sep).join('/');
|
|
349
|
+
const byKey = new Map(candidates.map(p => [keyOf(p), p]));
|
|
350
|
+
|
|
351
|
+
const deps = new Map();
|
|
352
|
+
for (const p of candidates) {
|
|
353
|
+
const set = new Set();
|
|
354
|
+
if (p.endsWith('.js')) {
|
|
355
|
+
const code = contents.get(p).toString('utf-8');
|
|
356
|
+
for (const m of code.matchAll(SPECIFIER_PATTERN)) {
|
|
357
|
+
const key = this.resolveSpecifierKey(m[2], p);
|
|
358
|
+
const target = key && byKey.get(key);
|
|
359
|
+
if (target && target !== p) set.add(target);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
deps.set(p, set);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const ordered = [];
|
|
366
|
+
const state = new Map();
|
|
367
|
+
const visit = (p) => {
|
|
368
|
+
if (state.has(p)) return;
|
|
369
|
+
state.set(p, true);
|
|
370
|
+
for (const d of deps.get(p)) visit(d);
|
|
371
|
+
ordered.push(p);
|
|
372
|
+
};
|
|
373
|
+
for (const p of candidates) visit(p);
|
|
374
|
+
return ordered;
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
// EXPORT WRAPPER
|
|
379
|
+
// -----------------------------
|
|
380
|
+
static async export(opts) {
|
|
381
|
+
return new FingerprintAssets(opts).init();
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
// EXPORT
|
|
387
|
+
// -----------------------------
|
|
388
|
+
export default FingerprintAssets.export;
|
|
@@ -26,9 +26,15 @@ import { distPath, srcPath, formats as formatsConfig } from '../utils/config/con
|
|
|
26
26
|
// DEFINE
|
|
27
27
|
// -----------------------------
|
|
28
28
|
class GenerateFormats {
|
|
29
|
-
constructor({ store, data }) {
|
|
29
|
+
constructor({ store, data, pages, distPath: distPathOverride }) {
|
|
30
30
|
this.store = store;
|
|
31
31
|
this.data = data || {};
|
|
32
|
+
// Optional per-call overrides
|
|
33
|
+
this.distPath = distPathOverride || distPath;
|
|
34
|
+
// Optional shared page snapshot (see scripts/utils/site-pages) —
|
|
35
|
+
// avoids re-walking dist and re-parsing every page. Doms are shared
|
|
36
|
+
// and must be treated as read-only.
|
|
37
|
+
this.pages = pages || null;
|
|
32
38
|
this.config = {
|
|
33
39
|
markdown: true,
|
|
34
40
|
json: true,
|
|
@@ -57,15 +63,19 @@ class GenerateFormats {
|
|
|
57
63
|
this.startLog();
|
|
58
64
|
|
|
59
65
|
try {
|
|
60
|
-
//
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
66
|
+
// Use the shared snapshot when provided; otherwise walk + parse
|
|
67
|
+
let pages = this.pages;
|
|
68
|
+
if (!pages) {
|
|
69
|
+
const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
|
|
70
|
+
const indexFiles = htmlFiles
|
|
71
|
+
.filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
|
|
72
|
+
pages = indexFiles.map(filePath => ({ filePath, urlPath: null, dom: null }));
|
|
73
|
+
}
|
|
74
|
+
pages = pages.filter(p => !p.filePath.includes('/404/'));
|
|
65
75
|
|
|
66
76
|
// Process each page
|
|
67
|
-
for (const
|
|
68
|
-
await this.processPage(
|
|
77
|
+
for (const page of pages) {
|
|
78
|
+
await this.processPage(page);
|
|
69
79
|
}
|
|
70
80
|
|
|
71
81
|
// Generate feed
|
|
@@ -75,15 +85,15 @@ class GenerateFormats {
|
|
|
75
85
|
|
|
76
86
|
// Generate llms.txt
|
|
77
87
|
if (this.config.llmsTxt) {
|
|
78
|
-
await this.generateLlmsTxt(
|
|
88
|
+
await this.generateLlmsTxt(pages);
|
|
79
89
|
}
|
|
80
90
|
|
|
81
91
|
// Generate llms-full.txt (complete documentation in one file)
|
|
82
92
|
if (this.config.llmsFullTxt) {
|
|
83
|
-
await this.generateLlmsFullTxt(
|
|
93
|
+
await this.generateLlmsFullTxt(pages);
|
|
84
94
|
}
|
|
85
95
|
} catch (err) {
|
|
86
|
-
Util.
|
|
96
|
+
Util.customWarn(err, 'generate-formats.js (formats skipped)');
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
// END LOGGING
|
|
@@ -92,13 +102,13 @@ class GenerateFormats {
|
|
|
92
102
|
|
|
93
103
|
/**
|
|
94
104
|
* Process a single page: generate markdown copy and JSON metadata
|
|
95
|
-
* @param {
|
|
105
|
+
* @param {Object} page - { filePath, urlPath?, dom? } snapshot entry
|
|
96
106
|
*/
|
|
97
|
-
async processPage(
|
|
107
|
+
async processPage(page) {
|
|
98
108
|
// Get the URL path from the dist file path
|
|
99
109
|
// e.g., dist/blog/my-post/index.html → /blog/my-post/
|
|
100
|
-
const
|
|
101
|
-
|
|
110
|
+
const urlPath = page.urlPath
|
|
111
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
102
112
|
|
|
103
113
|
// Try to find corresponding markdown source
|
|
104
114
|
const mdSourcePath = this.findMarkdownSource(urlPath);
|
|
@@ -110,7 +120,7 @@ class GenerateFormats {
|
|
|
110
120
|
|
|
111
121
|
// Generate JSON metadata
|
|
112
122
|
if (this.config.json) {
|
|
113
|
-
await this.generateJsonMeta(
|
|
123
|
+
await this.generateJsonMeta(page, urlPath, mdSourcePath);
|
|
114
124
|
}
|
|
115
125
|
}
|
|
116
126
|
|
|
@@ -146,14 +156,14 @@ class GenerateFormats {
|
|
|
146
156
|
const content = await fs.readFile(mdSourcePath, 'utf-8');
|
|
147
157
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
|
|
148
158
|
const outputPath = trimmedPath
|
|
149
|
-
? path.resolve(`${distPath}/md/${trimmedPath}/index.md`)
|
|
150
|
-
: path.resolve(`${distPath}/md/index.md`);
|
|
159
|
+
? path.resolve(`${this.distPath}/md/${trimmedPath}/index.md`)
|
|
160
|
+
: path.resolve(`${this.distPath}/md/index.md`);
|
|
151
161
|
|
|
152
162
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
153
163
|
await fs.writeFile(outputPath, content, 'utf-8');
|
|
154
164
|
this.stats.markdown++;
|
|
155
165
|
} catch (err) {
|
|
156
|
-
Util.
|
|
166
|
+
Util.customWarn(err, `generate-formats.js: markdown for ${urlPath} (skipped)`);
|
|
157
167
|
}
|
|
158
168
|
}
|
|
159
169
|
|
|
@@ -163,10 +173,10 @@ class GenerateFormats {
|
|
|
163
173
|
* @param {string} urlPath - URL path
|
|
164
174
|
* @param {string|null} mdSourcePath - Path to markdown source if exists
|
|
165
175
|
*/
|
|
166
|
-
async generateJsonMeta(
|
|
176
|
+
async generateJsonMeta(page, urlPath, mdSourcePath) {
|
|
167
177
|
try {
|
|
168
|
-
const
|
|
169
|
-
|
|
178
|
+
const dom = page.dom
|
|
179
|
+
|| Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
|
|
170
180
|
const document = dom.window.document;
|
|
171
181
|
|
|
172
182
|
// Extract metadata from the HTML
|
|
@@ -198,13 +208,13 @@ class GenerateFormats {
|
|
|
198
208
|
|
|
199
209
|
// Write JSON
|
|
200
210
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '') || 'index';
|
|
201
|
-
const outputPath = path.resolve(`${distPath}/api/${trimmedPath}.json`);
|
|
211
|
+
const outputPath = path.resolve(`${this.distPath}/api/${trimmedPath}.json`);
|
|
202
212
|
|
|
203
213
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
204
214
|
await fs.writeFile(outputPath, JSON.stringify(meta, null, 2), 'utf-8');
|
|
205
215
|
this.stats.json++;
|
|
206
216
|
} catch (err) {
|
|
207
|
-
Util.
|
|
217
|
+
Util.customWarn(err, `generate-formats.js: json for ${urlPath} (skipped)`);
|
|
208
218
|
}
|
|
209
219
|
}
|
|
210
220
|
|
|
@@ -273,7 +283,7 @@ class GenerateFormats {
|
|
|
273
283
|
xml += `</feed>
|
|
274
284
|
`;
|
|
275
285
|
|
|
276
|
-
await fs.writeFile(path.resolve(`${distPath}/feed.xml`), xml, 'utf-8');
|
|
286
|
+
await fs.writeFile(path.resolve(`${this.distPath}/feed.xml`), xml, 'utf-8');
|
|
277
287
|
this.stats.feed = true;
|
|
278
288
|
}
|
|
279
289
|
|
|
@@ -283,11 +293,11 @@ class GenerateFormats {
|
|
|
283
293
|
* @param {string[]} indexFiles - All index.html file paths
|
|
284
294
|
* @returns {Array<{title: string, description: string, url: string, mdUrl: string|null, section: string|null}>}
|
|
285
295
|
*/
|
|
286
|
-
async collectPageMeta(
|
|
296
|
+
async collectPageMeta(pageEntries) {
|
|
287
297
|
const pages = [];
|
|
288
|
-
for (const
|
|
289
|
-
const
|
|
290
|
-
|
|
298
|
+
for (const page of pageEntries) {
|
|
299
|
+
const urlPath = page.urlPath
|
|
300
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
291
301
|
const mdSource = this.findMarkdownSource(urlPath);
|
|
292
302
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
|
|
293
303
|
|
|
@@ -296,8 +306,8 @@ class GenerateFormats {
|
|
|
296
306
|
let description = '';
|
|
297
307
|
let section = null;
|
|
298
308
|
try {
|
|
299
|
-
const
|
|
300
|
-
|
|
309
|
+
const dom = page.dom
|
|
310
|
+
|| Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
|
|
301
311
|
const doc = dom.window.document;
|
|
302
312
|
const h1 = doc.querySelector('h1')?.textContent;
|
|
303
313
|
const titleTag = doc.querySelector('title')?.textContent;
|
|
@@ -328,7 +338,7 @@ class GenerateFormats {
|
|
|
328
338
|
* Enhanced with descriptions, section grouping, and AI instructions.
|
|
329
339
|
* @param {string[]} indexFiles - All index.html file paths
|
|
330
340
|
*/
|
|
331
|
-
async generateLlmsTxt(
|
|
341
|
+
async generateLlmsTxt(pageEntries) {
|
|
332
342
|
const siteTitle = this.data.siteTitle || 'Site';
|
|
333
343
|
const siteDescription = this.data.siteDescription || '';
|
|
334
344
|
const siteUrl = this.data.siteUrl || '';
|
|
@@ -343,7 +353,7 @@ class GenerateFormats {
|
|
|
343
353
|
}
|
|
344
354
|
|
|
345
355
|
// Collect page metadata
|
|
346
|
-
const pages = await this.collectPageMeta(
|
|
356
|
+
const pages = await this.collectPageMeta(pageEntries);
|
|
347
357
|
const mdPages = pages.filter(p => p.mdUrl);
|
|
348
358
|
|
|
349
359
|
if (mdPages.length) {
|
|
@@ -393,7 +403,7 @@ class GenerateFormats {
|
|
|
393
403
|
}
|
|
394
404
|
}
|
|
395
405
|
|
|
396
|
-
await fs.writeFile(path.resolve(`${distPath}/llms.txt`), content, 'utf-8');
|
|
406
|
+
await fs.writeFile(path.resolve(`${this.distPath}/llms.txt`), content, 'utf-8');
|
|
397
407
|
this.stats.llmsTxt = true;
|
|
398
408
|
}
|
|
399
409
|
|
|
@@ -402,7 +412,7 @@ class GenerateFormats {
|
|
|
402
412
|
* Ideal for bulk ingestion by IDE tools and AI systems.
|
|
403
413
|
* @param {string[]} indexFiles - All index.html file paths
|
|
404
414
|
*/
|
|
405
|
-
async generateLlmsFullTxt(
|
|
415
|
+
async generateLlmsFullTxt(pageEntries) {
|
|
406
416
|
const siteTitle = this.data.siteTitle || 'Site';
|
|
407
417
|
const siteDescription = this.data.siteDescription || '';
|
|
408
418
|
const llmsInstructions = this.data.llmsInstructions || '';
|
|
@@ -416,9 +426,9 @@ class GenerateFormats {
|
|
|
416
426
|
}
|
|
417
427
|
|
|
418
428
|
// Collect pages and read their markdown content
|
|
419
|
-
for (const
|
|
420
|
-
const
|
|
421
|
-
|
|
429
|
+
for (const page of pageEntries) {
|
|
430
|
+
const urlPath = page.urlPath
|
|
431
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
422
432
|
const mdSource = this.findMarkdownSource(urlPath);
|
|
423
433
|
if (!mdSource) continue;
|
|
424
434
|
|
|
@@ -439,7 +449,7 @@ class GenerateFormats {
|
|
|
439
449
|
} catch { /* skip unreadable files */ }
|
|
440
450
|
}
|
|
441
451
|
|
|
442
|
-
await fs.writeFile(path.resolve(`${distPath}/llms-full.txt`), content, 'utf-8');
|
|
452
|
+
await fs.writeFile(path.resolve(`${this.distPath}/llms-full.txt`), content, 'utf-8');
|
|
443
453
|
this.stats.llmsFullTxt = true;
|
|
444
454
|
}
|
|
445
455
|
|
|
@@ -22,8 +22,13 @@ import { distPath, fragments as fragmentsConfig } from '../utils/config/config.j
|
|
|
22
22
|
// DEFINE
|
|
23
23
|
// -----------------------------
|
|
24
24
|
class GenerateFragments {
|
|
25
|
-
constructor() {
|
|
25
|
+
constructor(opts = {}) {
|
|
26
26
|
this.total = 0;
|
|
27
|
+
// Optional per-call overrides
|
|
28
|
+
this.distPath = opts.distPath || distPath;
|
|
29
|
+
// Optional shared page snapshot (see scripts/utils/site-pages) —
|
|
30
|
+
// avoids re-walking dist and re-parsing every page
|
|
31
|
+
this.pages = opts.pages || null;
|
|
27
32
|
|
|
28
33
|
// Merge defaults with user config
|
|
29
34
|
this.config = {
|
|
@@ -50,15 +55,22 @@ class GenerateFragments {
|
|
|
50
55
|
this.startLog();
|
|
51
56
|
|
|
52
57
|
try {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
if (this.pages) {
|
|
59
|
+
// Shared snapshot: pages are already read and parsed
|
|
60
|
+
for (const page of this.pages) {
|
|
61
|
+
await this.generateFragment(page.filePath, page.dom);
|
|
62
|
+
}
|
|
63
|
+
} else {
|
|
64
|
+
// Standalone: walk and parse ourselves
|
|
65
|
+
const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
|
|
66
|
+
const indexFiles = htmlFiles.filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
|
|
67
|
+
|
|
68
|
+
for (const filePath of indexFiles) {
|
|
69
|
+
await this.generateFragment(filePath);
|
|
70
|
+
}
|
|
59
71
|
}
|
|
60
72
|
} catch (err) {
|
|
61
|
-
Util.
|
|
73
|
+
Util.customWarn(err, 'generate-fragments.js (fragments skipped)');
|
|
62
74
|
}
|
|
63
75
|
|
|
64
76
|
// END LOGGING
|
|
@@ -68,13 +80,11 @@ class GenerateFragments {
|
|
|
68
80
|
/**
|
|
69
81
|
* Generate a fragment file for a single HTML page
|
|
70
82
|
* @param {string} filePath - Path to the index.html file
|
|
83
|
+
* @param {Object} [sharedDom] - Pre-parsed document from the shared snapshot (read-only)
|
|
71
84
|
*/
|
|
72
|
-
async generateFragment(filePath) {
|
|
85
|
+
async generateFragment(filePath, sharedDom) {
|
|
73
86
|
try {
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
// Parse with JSDOM
|
|
77
|
-
const dom = Util.jsdom.dom({ src });
|
|
87
|
+
const dom = sharedDom || Util.jsdom.dom({ src: await fs.readFile(filePath, 'utf-8') });
|
|
78
88
|
const document = dom.window.document;
|
|
79
89
|
|
|
80
90
|
// Find the target element
|
|
@@ -91,7 +101,7 @@ class GenerateFragments {
|
|
|
91
101
|
|
|
92
102
|
this.total++;
|
|
93
103
|
} catch (err) {
|
|
94
|
-
Util.
|
|
104
|
+
Util.customWarn(err, `generate-fragments.js: ${filePath} (skipped)`);
|
|
95
105
|
}
|
|
96
106
|
}
|
|
97
107
|
|