cook-ssg 2.1.2 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/README.md +48 -20
  2. package/package.json +4 -7
  3. package/scripts/build.js +98 -26
  4. package/scripts/dev.js +46 -16
  5. package/scripts/plugins/apply-template.js +8 -17
  6. package/scripts/plugins/auto-components.js +2 -2
  7. package/scripts/plugins/bundle.js +6 -3
  8. package/scripts/plugins/copy-src.js +1 -1
  9. package/scripts/plugins/create-dir-from-file.js +4 -2
  10. package/scripts/plugins/custom-plugins.js +1 -1
  11. package/scripts/plugins/declarative-shadow-dom.js +2 -2
  12. package/scripts/plugins/escape-code-blocks.js +22 -1
  13. package/scripts/plugins/fingerprint-assets.js +388 -0
  14. package/scripts/plugins/generate-formats.js +49 -39
  15. package/scripts/plugins/generate-fragments.js +24 -14
  16. package/scripts/plugins/generate-sitemap-xml.js +38 -15
  17. package/scripts/plugins/minify-src.js +98 -72
  18. package/scripts/plugins/optimize-images.js +80 -26
  19. package/scripts/plugins/process-markdown.js +12 -25
  20. package/scripts/plugins/repeat-collection.js +70 -45
  21. package/scripts/plugins/replace-components.js +82 -60
  22. package/scripts/plugins/replace-external-link-protocol.js +16 -31
  23. package/scripts/plugins/replace-include.js +34 -19
  24. package/scripts/plugins/replace-inline.js +6 -3
  25. package/scripts/plugins/replace-template-strings.js +2 -50
  26. package/scripts/plugins/set-active-links.js +57 -26
  27. package/scripts/utils/config/config.js +44 -5
  28. package/scripts/utils/get-src/get-src.js +9 -45
  29. package/scripts/utils/interpolate/interpolate.js +83 -0
  30. package/scripts/utils/logger/logger.js +8 -3
  31. package/scripts/utils/site-pages/site-pages.js +37 -0
  32. package/scripts/utils/source-file/source-file.js +64 -0
  33. package/scripts/utils/user-data/user-data.js +45 -0
  34. package/scripts/utils/util/util.js +88 -46
  35. package/patches/jsdom-fetch-fix.js +0 -52
@@ -0,0 +1,388 @@
1
+ /**
2
+ * @file fingerprint-assets.js
3
+ * @description Content-hash fingerprinting for built CSS and JS assets.
4
+ *
5
+ * Runs as a post-pass over dist/ after all HTML and bundles are written.
6
+ *
7
+ * Phase 1: Hash and rename matching assets (foo.css -> foo.a1b2c3d4.css).
8
+ * Source map files (.map) are renamed alongside their source and the
9
+ * sourceMappingURL comment is updated to point to the new map name.
10
+ * Phase 2: Walk dist/ HTML and rewrite <link href> and <script src> refs.
11
+ * Phase 3: Emit dist/<manifestPath> mapping original -> hashed paths.
12
+ *
13
+ * Out of scope for v1: url() refs inside CSS, image fingerprinting.
14
+ */
15
+
16
+ // IMPORTS
17
+ // -----------------------------
18
+ import { createHash } from 'node:crypto';
19
+ import { promises as fs } from 'node:fs';
20
+ import path from 'node:path';
21
+ import Logger from '../utils/logger/logger.js';
22
+ import Util from '../utils/util/util.js';
23
+
24
+ // Config
25
+ import { distPath, fingerprint as fingerprintConfig } from '../utils/config/config.js';
26
+
27
+
28
+ // ============================================================================
29
+ // Pure helpers (extracted for unit testing)
30
+ // ============================================================================
31
+
32
+ /**
33
+ * Compute a content hash truncated to the given length.
34
+ * @param {Buffer|string} content
35
+ * @param {number} length - Number of hex chars to keep
36
+ * @param {string} [algorithm='sha256']
37
+ * @returns {string}
38
+ */
39
+ export function computeHash(content, length, algorithm = 'sha256') {
40
+ return createHash(algorithm).update(content).digest('hex').slice(0, length);
41
+ }
42
+
43
+ /**
44
+ * Insert a hash into a filename before the extension.
45
+ * Handles compound extensions like `.js.map` by inserting before the last
46
+ * meaningful extension (e.g. `foo.js.map` -> `foo.{hash}.js.map`).
47
+ * @param {string} filename - Base filename, e.g. "foo.css"
48
+ * @param {string} hash
49
+ * @returns {string} e.g. "foo.{hash}.css"
50
+ */
51
+ export function fingerprintFilename(filename, hash) {
52
+ // Map files: keep .js.map / .css.map paired with their source hash
53
+ if (filename.endsWith('.js.map')) {
54
+ const stem = filename.slice(0, -'.js.map'.length);
55
+ return `${stem}.${hash}.js.map`;
56
+ }
57
+ if (filename.endsWith('.css.map')) {
58
+ const stem = filename.slice(0, -'.css.map'.length);
59
+ return `${stem}.${hash}.css.map`;
60
+ }
61
+ const ext = path.extname(filename);
62
+ if (!ext) return `${filename}.${hash}`;
63
+ const stem = filename.slice(0, -ext.length);
64
+ return `${stem}.${hash}${ext}`;
65
+ }
66
+
67
+ /**
68
+ * Should this href/src be left alone?
69
+ * @param {string} url
70
+ * @returns {boolean}
71
+ */
72
+ export function isExternalRef(url) {
73
+ if (!url) return true;
74
+ if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('//')) return true;
75
+ if (url.startsWith('data:')) return true;
76
+ if (url.startsWith('mailto:') || url.startsWith('tel:')) return true;
77
+ return false;
78
+ }
79
+
80
+ /**
81
+ * Strip query string / fragment from a URL for manifest lookup.
82
+ * Returns { path, suffix } so the suffix can be reattached after rewriting.
83
+ * @param {string} url
84
+ */
85
+ export function splitUrlSuffix(url) {
86
+ const qIdx = url.search(/[?#]/);
87
+ if (qIdx === -1) return { path: url, suffix: '' };
88
+ return { path: url.slice(0, qIdx), suffix: url.slice(qIdx) };
89
+ }
90
+
91
+ /**
92
+ * Rewrite <link href> and <script src> in an HTML string using the manifest.
93
+ * Manifest keys are absolute-from-dist paths (e.g. "/assets/css/site.css").
94
+ * @param {string} html
95
+ * @param {Object} manifest - { "/assets/css/site.css": "/assets/css/site.{hash}.css", ... }
96
+ * @returns {{ html: string, rewrites: number }}
97
+ */
98
+ export function rewriteHrefs(html, manifest) {
99
+ if (!manifest || Object.keys(manifest).length === 0) return { html, rewrites: 0 };
100
+
101
+ // linkedom round-trips exactly what was parsed: fragments stay
102
+ // fragments (real JSDOM wrapped them in <html><head><body>, corrupting
103
+ // _fragment.html files consumed by SPA navigation).
104
+ const dom = Util.jsdom.dom({ src: html });
105
+ const document = dom.window.document;
106
+ let rewrites = 0;
107
+
108
+ const rewriteAttr = (el, attr) => {
109
+ const original = el.getAttribute(attr);
110
+ if (!original || isExternalRef(original)) return;
111
+ const { path: cleanPath, suffix } = splitUrlSuffix(original);
112
+ const lookupKey = cleanPath.startsWith('/') ? cleanPath : `/${cleanPath}`;
113
+ const replacement = manifest[lookupKey];
114
+ if (!replacement) return;
115
+ // Preserve leading-slash style of the original
116
+ const newValue = cleanPath.startsWith('/') ? replacement : replacement.replace(/^\//, '');
117
+ el.setAttribute(attr, `${newValue}${suffix}`);
118
+ rewrites += 1;
119
+ };
120
+
121
+ document.querySelectorAll('link[href]').forEach(el => rewriteAttr(el, 'href'));
122
+ document.querySelectorAll('script[src]').forEach(el => rewriteAttr(el, 'src'));
123
+
124
+ if (rewrites === 0) return { html, rewrites: 0 };
125
+ return { html: Util.setSrc({ dom }), rewrites };
126
+ }
127
+
128
+ /**
129
+ * Matches ES module specifiers: `import x from '...'`, `export ... from '...'`,
130
+ * side-effect `import '...'`, and dynamic `import('...')`.
131
+ */
132
+ const SPECIFIER_PATTERN = /((?:\bfrom|\bimport)\s*\(?\s*['"])([^'"]+)(['"])/g;
133
+
134
+
135
+ // ============================================================================
136
+ // Plugin
137
+ // ============================================================================
138
+
139
+ class FingerprintAssets {
140
+ constructor(opts = {}) {
141
+ // Optional per-call overrides (used by tests and custom pipelines)
142
+ this.distPath = opts.distPath || distPath;
143
+ this.config = {
144
+ enabled: false,
145
+ hashLength: 8,
146
+ algorithm: 'sha256',
147
+ types: ['.css', '.js'],
148
+ excludePaths: [],
149
+ manifestPath: 'asset-manifest.json',
150
+ ...fingerprintConfig,
151
+ ...(opts.fingerprint || {}),
152
+ };
153
+ }
154
+
155
+ async init() {
156
+ if (!this.config.enabled) return;
157
+ if (process.env.DEV_CHANGED_PAGE) return;
158
+
159
+ Logger.persist.header('\nFingerprint Assets');
160
+
161
+ // Phase 1: Find candidates and hash + rename
162
+ const candidates = await this.findCandidates();
163
+ if (!candidates.length) {
164
+ Logger.persist.success('No matching assets (skipped)');
165
+ return;
166
+ }
167
+
168
+ const manifest = await this.fingerprintFiles(candidates);
169
+
170
+ // Phase 2: Rewrite HTML references
171
+ const htmlFiles = (Util.getPaths(this.distPath, this.distPath, []) || [])
172
+ .filter(p => p.endsWith('.html'));
173
+ let totalRewrites = 0;
174
+ for (const htmlPath of htmlFiles) {
175
+ const original = await fs.readFile(htmlPath, 'utf-8');
176
+ const { html, rewrites } = rewriteHrefs(original, manifest);
177
+ if (rewrites > 0) {
178
+ await fs.writeFile(htmlPath, html, 'utf-8');
179
+ totalRewrites += rewrites;
180
+ }
181
+ }
182
+
183
+ // Phase 3: Write manifest
184
+ const manifestFsPath = path.join(this.distPath, this.config.manifestPath);
185
+ await fs.writeFile(manifestFsPath, JSON.stringify(manifest, null, 2), 'utf-8');
186
+
187
+ Logger.persist.success(
188
+ `Fingerprinted ${candidates.length} asset(s), rewrote ${totalRewrites} HTML ref(s)`
189
+ );
190
+ }
191
+
192
+ /**
193
+ * Find all dist files matching configured types, excluding configured paths.
194
+ * Returns absolute paths.
195
+ */
196
+ async findCandidates() {
197
+ const all = Util.getPaths(this.distPath, this.distPath, []) || [];
198
+ const excludes = this.config.excludePaths.map(p => (p instanceof RegExp ? p : new RegExp(p)));
199
+ return all.filter(p => {
200
+ // Skip source maps in the candidate list — handled alongside their source.
201
+ if (p.endsWith('.map')) return false;
202
+ const ext = path.extname(p);
203
+ if (!this.config.types.includes(ext)) return false;
204
+ if (excludes.some(r => r.test(p))) return false;
205
+ return true;
206
+ });
207
+ }
208
+
209
+ /**
210
+ * Hash, rename, and (if present) rename the paired .map and patch sourceMappingURL.
211
+ * @param {string[]} candidates - Absolute paths
212
+ * @returns {Promise<Object>} manifest of relative-from-dist mappings
213
+ */
214
+ async fingerprintFiles(candidates) {
215
+ const manifest = {};
216
+
217
+ // Read all candidates up front so we can order JS files by their
218
+ // import graph: a dependency must be hashed before its importers,
219
+ // because the importer's content embeds the dependency's hashed name.
220
+ // Otherwise an importer keeps its old hash when a dependency changes,
221
+ // and immutable caching serves a stale importer pointing at a dead file.
222
+ const contents = new Map();
223
+ for (const p of candidates) contents.set(p, await fs.readFile(p));
224
+ const ordered = this.orderByDependencies(candidates, contents);
225
+
226
+ for (const absPath of ordered) {
227
+ let buffer = contents.get(absPath);
228
+ let contentChanged = false;
229
+
230
+ // Rewrite import specifiers of already-fingerprinted dependencies
231
+ // BEFORE hashing, so this file's hash reflects theirs
232
+ if (absPath.endsWith('.js')) {
233
+ const { code, changed } = this.rewriteImportSpecifiers(
234
+ buffer.toString('utf-8'), absPath, manifest
235
+ );
236
+ if (changed) {
237
+ buffer = Buffer.from(code, 'utf-8');
238
+ contentChanged = true;
239
+ }
240
+ }
241
+
242
+ const hash = computeHash(buffer, this.config.hashLength, this.config.algorithm);
243
+
244
+ const dir = path.dirname(absPath);
245
+ const baseName = path.basename(absPath);
246
+ const newName = fingerprintFilename(baseName, hash);
247
+ const newAbs = path.join(dir, newName);
248
+
249
+ // Handle paired .map file (if present)
250
+ const mapAbs = `${absPath}.map`;
251
+ let updatedContent = null;
252
+ let mapNewName = null;
253
+ try {
254
+ await fs.access(mapAbs);
255
+ // Rename map file with the same hash
256
+ mapNewName = fingerprintFilename(`${baseName}.map`, hash);
257
+ const mapNewAbs = path.join(dir, mapNewName);
258
+ await fs.rename(mapAbs, mapNewAbs);
259
+ // Patch sourceMappingURL comment to point at the new map name
260
+ const text = buffer.toString('utf-8');
261
+ contentChanged = false; // superseded by updatedContent below
262
+ updatedContent = text.replace(
263
+ /(sourceMappingURL=)([^\s*'"]+)/,
264
+ (_, prefix) => `${prefix}${mapNewName}`
265
+ );
266
+ } catch {
267
+ // No map file
268
+ }
269
+
270
+ if (updatedContent !== null) {
271
+ await fs.writeFile(newAbs, updatedContent, 'utf-8');
272
+ await fs.unlink(absPath);
273
+ } else if (contentChanged) {
274
+ // Import specifiers were rewritten — write the updated content
275
+ await fs.writeFile(newAbs, buffer);
276
+ await fs.unlink(absPath);
277
+ } else {
278
+ await fs.rename(absPath, newAbs);
279
+ }
280
+
281
+ // Build manifest keys as web paths starting with `/`
282
+ const relOriginal = '/' + path.relative(this.distPath, absPath).split(path.sep).join('/');
283
+ const relNew = '/' + path.relative(this.distPath, newAbs).split(path.sep).join('/');
284
+ manifest[relOriginal] = relNew;
285
+
286
+ if (mapNewName) {
287
+ const relMapOriginal = '/' + path.relative(this.distPath, mapAbs).split(path.sep).join('/');
288
+ const relMapNew = '/' + path.relative(this.distPath, path.join(dir, mapNewName)).split(path.sep).join('/');
289
+ manifest[relMapOriginal] = relMapNew;
290
+ }
291
+ }
292
+
293
+ return manifest;
294
+ }
295
+
296
+ /**
297
+ * Resolve an import specifier to a manifest key (absolute-from-dist web
298
+ * path), or null when it can't be fingerprinted (external URL, bare
299
+ * package specifier).
300
+ * @param {string} spec - Raw specifier from the source
301
+ * @param {string} absPath - File the specifier appears in
302
+ * @returns {string|null}
303
+ */
304
+ resolveSpecifierKey(spec, absPath) {
305
+ if (isExternalRef(spec)) return null;
306
+ const clean = splitUrlSuffix(spec).path;
307
+ if (clean.startsWith('/')) return clean;
308
+ if (clean.startsWith('./') || clean.startsWith('../')) {
309
+ const dirRel = path.dirname(path.relative(this.distPath, absPath));
310
+ const joined = path.normalize(path.join(dirRel, clean));
311
+ return '/' + joined.split(path.sep).join('/');
312
+ }
313
+ // Bare specifier (npm package / import map) — leave alone
314
+ return null;
315
+ }
316
+
317
+ /**
318
+ * Rewrite import/export/dynamic-import specifiers whose targets exist in
319
+ * the manifest. Fingerprinting only changes the basename, so the
320
+ * specifier's directory part is preserved as written.
321
+ * @param {string} code
322
+ * @param {string} absPath - File the code came from
323
+ * @param {Object} manifest
324
+ * @returns {{ code: string, changed: boolean }}
325
+ */
326
+ rewriteImportSpecifiers(code, absPath, manifest) {
327
+ let changed = false;
328
+ const out = code.replace(SPECIFIER_PATTERN, (match, pre, spec, post) => {
329
+ const key = this.resolveSpecifierKey(spec, absPath);
330
+ if (!key || !manifest[key]) return match;
331
+ const newBase = path.basename(manifest[key]);
332
+ const slashIdx = spec.lastIndexOf('/');
333
+ const newSpec = slashIdx === -1 ? newBase : spec.slice(0, slashIdx + 1) + newBase;
334
+ changed = true;
335
+ return `${pre}${newSpec}${post}`;
336
+ });
337
+ return { code: out, changed };
338
+ }
339
+
340
+ /**
341
+ * Order candidates so JS dependencies precede their importers
342
+ * (depth-first topological sort, cycle-tolerant).
343
+ * @param {string[]} candidates - Absolute paths
344
+ * @param {Map<string,Buffer>} contents
345
+ * @returns {string[]}
346
+ */
347
+ orderByDependencies(candidates, contents) {
348
+ const keyOf = p => '/' + path.relative(this.distPath, p).split(path.sep).join('/');
349
+ const byKey = new Map(candidates.map(p => [keyOf(p), p]));
350
+
351
+ const deps = new Map();
352
+ for (const p of candidates) {
353
+ const set = new Set();
354
+ if (p.endsWith('.js')) {
355
+ const code = contents.get(p).toString('utf-8');
356
+ for (const m of code.matchAll(SPECIFIER_PATTERN)) {
357
+ const key = this.resolveSpecifierKey(m[2], p);
358
+ const target = key && byKey.get(key);
359
+ if (target && target !== p) set.add(target);
360
+ }
361
+ }
362
+ deps.set(p, set);
363
+ }
364
+
365
+ const ordered = [];
366
+ const state = new Map();
367
+ const visit = (p) => {
368
+ if (state.has(p)) return;
369
+ state.set(p, true);
370
+ for (const d of deps.get(p)) visit(d);
371
+ ordered.push(p);
372
+ };
373
+ for (const p of candidates) visit(p);
374
+ return ordered;
375
+ }
376
+
377
+
378
+ // EXPORT WRAPPER
379
+ // -----------------------------
380
+ static async export(opts) {
381
+ return new FingerprintAssets(opts).init();
382
+ }
383
+ }
384
+
385
+
386
+ // EXPORT
387
+ // -----------------------------
388
+ export default FingerprintAssets.export;
@@ -26,9 +26,15 @@ import { distPath, srcPath, formats as formatsConfig } from '../utils/config/con
26
26
  // DEFINE
27
27
  // -----------------------------
28
28
  class GenerateFormats {
29
- constructor({ store, data }) {
29
+ constructor({ store, data, pages, distPath: distPathOverride }) {
30
30
  this.store = store;
31
31
  this.data = data || {};
32
+ // Optional per-call overrides
33
+ this.distPath = distPathOverride || distPath;
34
+ // Optional shared page snapshot (see scripts/utils/site-pages) —
35
+ // avoids re-walking dist and re-parsing every page. Doms are shared
36
+ // and must be treated as read-only.
37
+ this.pages = pages || null;
32
38
  this.config = {
33
39
  markdown: true,
34
40
  json: true,
@@ -57,15 +63,19 @@ class GenerateFormats {
57
63
  this.startLog();
58
64
 
59
65
  try {
60
- // Get all index.html files in dist (the actual pages)
61
- const htmlFiles = Util.getPaths(distPath, distPath, []);
62
- const indexFiles = htmlFiles
63
- .filter(f => f.endsWith('/index.html') || f === `${distPath}/index.html`)
64
- .filter(f => !f.includes('/404/'));
66
+ // Use the shared snapshot when provided; otherwise walk + parse
67
+ let pages = this.pages;
68
+ if (!pages) {
69
+ const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
70
+ const indexFiles = htmlFiles
71
+ .filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
72
+ pages = indexFiles.map(filePath => ({ filePath, urlPath: null, dom: null }));
73
+ }
74
+ pages = pages.filter(p => !p.filePath.includes('/404/'));
65
75
 
66
76
  // Process each page
67
- for (const filePath of indexFiles) {
68
- await this.processPage(filePath);
77
+ for (const page of pages) {
78
+ await this.processPage(page);
69
79
  }
70
80
 
71
81
  // Generate feed
@@ -75,15 +85,15 @@ class GenerateFormats {
75
85
 
76
86
  // Generate llms.txt
77
87
  if (this.config.llmsTxt) {
78
- await this.generateLlmsTxt(indexFiles);
88
+ await this.generateLlmsTxt(pages);
79
89
  }
80
90
 
81
91
  // Generate llms-full.txt (complete documentation in one file)
82
92
  if (this.config.llmsFullTxt) {
83
- await this.generateLlmsFullTxt(indexFiles);
93
+ await this.generateLlmsFullTxt(pages);
84
94
  }
85
95
  } catch (err) {
86
- Util.customError(err, 'generate-formats.js');
96
+ Util.customWarn(err, 'generate-formats.js (formats skipped)');
87
97
  }
88
98
 
89
99
  // END LOGGING
@@ -92,13 +102,13 @@ class GenerateFormats {
92
102
 
93
103
  /**
94
104
  * Process a single page: generate markdown copy and JSON metadata
95
- * @param {string} filePath - Path to the index.html file in dist
105
+ * @param {Object} page - { filePath, urlPath?, dom? } snapshot entry
96
106
  */
97
- async processPage(filePath) {
107
+ async processPage(page) {
98
108
  // Get the URL path from the dist file path
99
109
  // e.g., dist/blog/my-post/index.html → /blog/my-post/
100
- const relativePath = filePath.replace(distPath, '');
101
- const urlPath = relativePath.replace(/\/index\.html$/, '/').replace(/^$/, '/');
110
+ const urlPath = page.urlPath
111
+ || page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
102
112
 
103
113
  // Try to find corresponding markdown source
104
114
  const mdSourcePath = this.findMarkdownSource(urlPath);
@@ -110,7 +120,7 @@ class GenerateFormats {
110
120
 
111
121
  // Generate JSON metadata
112
122
  if (this.config.json) {
113
- await this.generateJsonMeta(filePath, urlPath, mdSourcePath);
123
+ await this.generateJsonMeta(page, urlPath, mdSourcePath);
114
124
  }
115
125
  }
116
126
 
@@ -146,14 +156,14 @@ class GenerateFormats {
146
156
  const content = await fs.readFile(mdSourcePath, 'utf-8');
147
157
  const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
148
158
  const outputPath = trimmedPath
149
- ? path.resolve(`${distPath}/md/${trimmedPath}/index.md`)
150
- : path.resolve(`${distPath}/md/index.md`);
159
+ ? path.resolve(`${this.distPath}/md/${trimmedPath}/index.md`)
160
+ : path.resolve(`${this.distPath}/md/index.md`);
151
161
 
152
162
  await fs.mkdir(path.dirname(outputPath), { recursive: true });
153
163
  await fs.writeFile(outputPath, content, 'utf-8');
154
164
  this.stats.markdown++;
155
165
  } catch (err) {
156
- Util.customError(err, `generate-formats.js: markdown for ${urlPath}`);
166
+ Util.customWarn(err, `generate-formats.js: markdown for ${urlPath} (skipped)`);
157
167
  }
158
168
  }
159
169
 
@@ -163,10 +173,10 @@ class GenerateFormats {
163
173
  * @param {string} urlPath - URL path
164
174
  * @param {string|null} mdSourcePath - Path to markdown source if exists
165
175
  */
166
- async generateJsonMeta(filePath, urlPath, mdSourcePath) {
176
+ async generateJsonMeta(page, urlPath, mdSourcePath) {
167
177
  try {
168
- const src = await fs.readFile(filePath, 'utf-8');
169
- const dom = Util.jsdom.dom({ src });
178
+ const dom = page.dom
179
+ || Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
170
180
  const document = dom.window.document;
171
181
 
172
182
  // Extract metadata from the HTML
@@ -198,13 +208,13 @@ class GenerateFormats {
198
208
 
199
209
  // Write JSON
200
210
  const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '') || 'index';
201
- const outputPath = path.resolve(`${distPath}/api/${trimmedPath}.json`);
211
+ const outputPath = path.resolve(`${this.distPath}/api/${trimmedPath}.json`);
202
212
 
203
213
  await fs.mkdir(path.dirname(outputPath), { recursive: true });
204
214
  await fs.writeFile(outputPath, JSON.stringify(meta, null, 2), 'utf-8');
205
215
  this.stats.json++;
206
216
  } catch (err) {
207
- Util.customError(err, `generate-formats.js: json for ${urlPath}`);
217
+ Util.customWarn(err, `generate-formats.js: json for ${urlPath} (skipped)`);
208
218
  }
209
219
  }
210
220
 
@@ -273,7 +283,7 @@ class GenerateFormats {
273
283
  xml += `</feed>
274
284
  `;
275
285
 
276
- await fs.writeFile(path.resolve(`${distPath}/feed.xml`), xml, 'utf-8');
286
+ await fs.writeFile(path.resolve(`${this.distPath}/feed.xml`), xml, 'utf-8');
277
287
  this.stats.feed = true;
278
288
  }
279
289
 
@@ -283,11 +293,11 @@ class GenerateFormats {
283
293
  * @param {string[]} indexFiles - All index.html file paths
284
294
  * @returns {Array<{title: string, description: string, url: string, mdUrl: string|null, section: string|null}>}
285
295
  */
286
- async collectPageMeta(indexFiles) {
296
+ async collectPageMeta(pageEntries) {
287
297
  const pages = [];
288
- for (const filePath of indexFiles) {
289
- const relativePath = filePath.replace(distPath, '');
290
- const urlPath = relativePath.replace(/\/index\.html$/, '/').replace(/^$/, '/');
298
+ for (const page of pageEntries) {
299
+ const urlPath = page.urlPath
300
+ || page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
291
301
  const mdSource = this.findMarkdownSource(urlPath);
292
302
  const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
293
303
 
@@ -296,8 +306,8 @@ class GenerateFormats {
296
306
  let description = '';
297
307
  let section = null;
298
308
  try {
299
- const src = await fs.readFile(filePath, 'utf-8');
300
- const dom = Util.jsdom.dom({ src });
309
+ const dom = page.dom
310
+ || Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
301
311
  const doc = dom.window.document;
302
312
  const h1 = doc.querySelector('h1')?.textContent;
303
313
  const titleTag = doc.querySelector('title')?.textContent;
@@ -328,7 +338,7 @@ class GenerateFormats {
328
338
  * Enhanced with descriptions, section grouping, and AI instructions.
329
339
  * @param {string[]} indexFiles - All index.html file paths
330
340
  */
331
- async generateLlmsTxt(indexFiles) {
341
+ async generateLlmsTxt(pageEntries) {
332
342
  const siteTitle = this.data.siteTitle || 'Site';
333
343
  const siteDescription = this.data.siteDescription || '';
334
344
  const siteUrl = this.data.siteUrl || '';
@@ -343,7 +353,7 @@ class GenerateFormats {
343
353
  }
344
354
 
345
355
  // Collect page metadata
346
- const pages = await this.collectPageMeta(indexFiles);
356
+ const pages = await this.collectPageMeta(pageEntries);
347
357
  const mdPages = pages.filter(p => p.mdUrl);
348
358
 
349
359
  if (mdPages.length) {
@@ -393,7 +403,7 @@ class GenerateFormats {
393
403
  }
394
404
  }
395
405
 
396
- await fs.writeFile(path.resolve(`${distPath}/llms.txt`), content, 'utf-8');
406
+ await fs.writeFile(path.resolve(`${this.distPath}/llms.txt`), content, 'utf-8');
397
407
  this.stats.llmsTxt = true;
398
408
  }
399
409
 
@@ -402,7 +412,7 @@ class GenerateFormats {
402
412
  * Ideal for bulk ingestion by IDE tools and AI systems.
403
413
  * @param {string[]} indexFiles - All index.html file paths
404
414
  */
405
- async generateLlmsFullTxt(indexFiles) {
415
+ async generateLlmsFullTxt(pageEntries) {
406
416
  const siteTitle = this.data.siteTitle || 'Site';
407
417
  const siteDescription = this.data.siteDescription || '';
408
418
  const llmsInstructions = this.data.llmsInstructions || '';
@@ -416,9 +426,9 @@ class GenerateFormats {
416
426
  }
417
427
 
418
428
  // Collect pages and read their markdown content
419
- for (const filePath of indexFiles) {
420
- const relativePath = filePath.replace(distPath, '');
421
- const urlPath = relativePath.replace(/\/index\.html$/, '/').replace(/^$/, '/');
429
+ for (const page of pageEntries) {
430
+ const urlPath = page.urlPath
431
+ || page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
422
432
  const mdSource = this.findMarkdownSource(urlPath);
423
433
  if (!mdSource) continue;
424
434
 
@@ -439,7 +449,7 @@ class GenerateFormats {
439
449
  } catch { /* skip unreadable files */ }
440
450
  }
441
451
 
442
- await fs.writeFile(path.resolve(`${distPath}/llms-full.txt`), content, 'utf-8');
452
+ await fs.writeFile(path.resolve(`${this.distPath}/llms-full.txt`), content, 'utf-8');
443
453
  this.stats.llmsFullTxt = true;
444
454
  }
445
455
 
@@ -22,8 +22,13 @@ import { distPath, fragments as fragmentsConfig } from '../utils/config/config.j
22
22
  // DEFINE
23
23
  // -----------------------------
24
24
  class GenerateFragments {
25
- constructor() {
25
+ constructor(opts = {}) {
26
26
  this.total = 0;
27
+ // Optional per-call overrides
28
+ this.distPath = opts.distPath || distPath;
29
+ // Optional shared page snapshot (see scripts/utils/site-pages) —
30
+ // avoids re-walking dist and re-parsing every page
31
+ this.pages = opts.pages || null;
27
32
 
28
33
  // Merge defaults with user config
29
34
  this.config = {
@@ -50,15 +55,22 @@ class GenerateFragments {
50
55
  this.startLog();
51
56
 
52
57
  try {
53
- // Get all index.html files in dist
54
- const htmlFiles = Util.getPaths(distPath, distPath, []);
55
- const indexFiles = htmlFiles.filter(f => f.endsWith('/index.html') || f === `${distPath}/index.html`);
56
-
57
- for (const filePath of indexFiles) {
58
- await this.generateFragment(filePath);
58
+ if (this.pages) {
59
+ // Shared snapshot: pages are already read and parsed
60
+ for (const page of this.pages) {
61
+ await this.generateFragment(page.filePath, page.dom);
62
+ }
63
+ } else {
64
+ // Standalone: walk and parse ourselves
65
+ const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
66
+ const indexFiles = htmlFiles.filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
67
+
68
+ for (const filePath of indexFiles) {
69
+ await this.generateFragment(filePath);
70
+ }
59
71
  }
60
72
  } catch (err) {
61
- Util.customError(err, 'generate-fragments.js');
73
+ Util.customWarn(err, 'generate-fragments.js (fragments skipped)');
62
74
  }
63
75
 
64
76
  // END LOGGING
@@ -68,13 +80,11 @@ class GenerateFragments {
68
80
  /**
69
81
  * Generate a fragment file for a single HTML page
70
82
  * @param {string} filePath - Path to the index.html file
83
+ * @param {Object} [sharedDom] - Pre-parsed document from the shared snapshot (read-only)
71
84
  */
72
- async generateFragment(filePath) {
85
+ async generateFragment(filePath, sharedDom) {
73
86
  try {
74
- const src = await fs.readFile(filePath, 'utf-8');
75
-
76
- // Parse with JSDOM
77
- const dom = Util.jsdom.dom({ src });
87
+ const dom = sharedDom || Util.jsdom.dom({ src: await fs.readFile(filePath, 'utf-8') });
78
88
  const document = dom.window.document;
79
89
 
80
90
  // Find the target element
@@ -91,7 +101,7 @@ class GenerateFragments {
91
101
 
92
102
  this.total++;
93
103
  } catch (err) {
94
- Util.customError(err, `generate-fragments.js: ${filePath}`);
104
+ Util.customWarn(err, `generate-fragments.js: ${filePath} (skipped)`);
95
105
  }
96
106
  }
97
107