cook-ssg 2.1.3 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -20
- package/package.json +4 -9
- package/scripts/build.js +91 -26
- package/scripts/dev.js +46 -16
- package/scripts/plugins/apply-template.js +8 -17
- package/scripts/plugins/auto-components.js +2 -2
- package/scripts/plugins/bundle.js +6 -3
- package/scripts/plugins/copy-src.js +1 -1
- package/scripts/plugins/create-dir-from-file.js +4 -2
- package/scripts/plugins/create-dist.js +1 -2
- package/scripts/plugins/custom-plugins.js +1 -1
- package/scripts/plugins/declarative-shadow-dom.js +2 -2
- package/scripts/plugins/escape-code-blocks.js +22 -1
- package/scripts/plugins/fingerprint-assets.js +133 -13
- package/scripts/plugins/generate-formats.js +49 -39
- package/scripts/plugins/generate-fragments.js +24 -14
- package/scripts/plugins/generate-sitemap-xml.js +38 -15
- package/scripts/plugins/minify-src.js +98 -72
- package/scripts/plugins/optimize-images.js +80 -26
- package/scripts/plugins/process-markdown.js +12 -25
- package/scripts/plugins/repeat-collection.js +70 -45
- package/scripts/plugins/replace-components.js +82 -60
- package/scripts/plugins/replace-external-link-protocol.js +16 -31
- package/scripts/plugins/replace-include.js +34 -19
- package/scripts/plugins/replace-inline.js +6 -3
- package/scripts/plugins/replace-template-strings.js +2 -50
- package/scripts/plugins/set-active-links.js +57 -26
- package/scripts/utils/config/config.js +29 -5
- package/scripts/utils/get-src/get-src.js +9 -45
- package/scripts/utils/interpolate/interpolate.js +83 -0
- package/scripts/utils/logger/logger.js +8 -3
- package/scripts/utils/site-pages/site-pages.js +37 -0
- package/scripts/utils/source-file/source-file.js +64 -0
- package/scripts/utils/user-data/user-data.js +45 -0
- package/scripts/utils/util/util.js +88 -46
- package/patches/jsdom-fetch-fix.js +0 -52
|
@@ -18,7 +18,6 @@
|
|
|
18
18
|
import { createHash } from 'node:crypto';
|
|
19
19
|
import { promises as fs } from 'node:fs';
|
|
20
20
|
import path from 'node:path';
|
|
21
|
-
import { JSDOM } from 'jsdom';
|
|
22
21
|
import Logger from '../utils/logger/logger.js';
|
|
23
22
|
import Util from '../utils/util/util.js';
|
|
24
23
|
|
|
@@ -99,7 +98,10 @@ export function splitUrlSuffix(url) {
|
|
|
99
98
|
export function rewriteHrefs(html, manifest) {
|
|
100
99
|
if (!manifest || Object.keys(manifest).length === 0) return { html, rewrites: 0 };
|
|
101
100
|
|
|
102
|
-
|
|
101
|
+
// linkedom round-trips exactly what was parsed: fragments stay
|
|
102
|
+
// fragments (real JSDOM wrapped them in <html><head><body>, corrupting
|
|
103
|
+
// _fragment.html files consumed by SPA navigation).
|
|
104
|
+
const dom = Util.jsdom.dom({ src: html });
|
|
103
105
|
const document = dom.window.document;
|
|
104
106
|
let rewrites = 0;
|
|
105
107
|
|
|
@@ -120,16 +122,24 @@ export function rewriteHrefs(html, manifest) {
|
|
|
120
122
|
document.querySelectorAll('script[src]').forEach(el => rewriteAttr(el, 'src'));
|
|
121
123
|
|
|
122
124
|
if (rewrites === 0) return { html, rewrites: 0 };
|
|
123
|
-
return { html:
|
|
125
|
+
return { html: Util.setSrc({ dom }), rewrites };
|
|
124
126
|
}
|
|
125
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Matches ES module specifiers: `import x from '...'`, `export ... from '...'`,
|
|
130
|
+
* side-effect `import '...'`, and dynamic `import('...')`.
|
|
131
|
+
*/
|
|
132
|
+
const SPECIFIER_PATTERN = /((?:\bfrom|\bimport)\s*\(?\s*['"])([^'"]+)(['"])/g;
|
|
133
|
+
|
|
126
134
|
|
|
127
135
|
// ============================================================================
|
|
128
136
|
// Plugin
|
|
129
137
|
// ============================================================================
|
|
130
138
|
|
|
131
139
|
class FingerprintAssets {
|
|
132
|
-
constructor() {
|
|
140
|
+
constructor(opts = {}) {
|
|
141
|
+
// Optional per-call overrides (used by tests and custom pipelines)
|
|
142
|
+
this.distPath = opts.distPath || distPath;
|
|
133
143
|
this.config = {
|
|
134
144
|
enabled: false,
|
|
135
145
|
hashLength: 8,
|
|
@@ -138,6 +148,7 @@ class FingerprintAssets {
|
|
|
138
148
|
excludePaths: [],
|
|
139
149
|
manifestPath: 'asset-manifest.json',
|
|
140
150
|
...fingerprintConfig,
|
|
151
|
+
...(opts.fingerprint || {}),
|
|
141
152
|
};
|
|
142
153
|
}
|
|
143
154
|
|
|
@@ -157,7 +168,7 @@ class FingerprintAssets {
|
|
|
157
168
|
const manifest = await this.fingerprintFiles(candidates);
|
|
158
169
|
|
|
159
170
|
// Phase 2: Rewrite HTML references
|
|
160
|
-
const htmlFiles = (Util.getPaths(distPath, distPath, []) || [])
|
|
171
|
+
const htmlFiles = (Util.getPaths(this.distPath, this.distPath, []) || [])
|
|
161
172
|
.filter(p => p.endsWith('.html'));
|
|
162
173
|
let totalRewrites = 0;
|
|
163
174
|
for (const htmlPath of htmlFiles) {
|
|
@@ -170,7 +181,7 @@ class FingerprintAssets {
|
|
|
170
181
|
}
|
|
171
182
|
|
|
172
183
|
// Phase 3: Write manifest
|
|
173
|
-
const manifestFsPath = path.join(distPath, this.config.manifestPath);
|
|
184
|
+
const manifestFsPath = path.join(this.distPath, this.config.manifestPath);
|
|
174
185
|
await fs.writeFile(manifestFsPath, JSON.stringify(manifest, null, 2), 'utf-8');
|
|
175
186
|
|
|
176
187
|
Logger.persist.success(
|
|
@@ -183,7 +194,7 @@ class FingerprintAssets {
|
|
|
183
194
|
* Returns absolute paths.
|
|
184
195
|
*/
|
|
185
196
|
async findCandidates() {
|
|
186
|
-
const all = Util.getPaths(distPath, distPath, []) || [];
|
|
197
|
+
const all = Util.getPaths(this.distPath, this.distPath, []) || [];
|
|
187
198
|
const excludes = this.config.excludePaths.map(p => (p instanceof RegExp ? p : new RegExp(p)));
|
|
188
199
|
return all.filter(p => {
|
|
189
200
|
// Skip source maps in the candidate list — handled alongside their source.
|
|
@@ -203,8 +214,31 @@ class FingerprintAssets {
|
|
|
203
214
|
async fingerprintFiles(candidates) {
|
|
204
215
|
const manifest = {};
|
|
205
216
|
|
|
206
|
-
|
|
207
|
-
|
|
217
|
+
// Read all candidates up front so we can order JS files by their
|
|
218
|
+
// import graph: a dependency must be hashed before its importers,
|
|
219
|
+
// because the importer's content embeds the dependency's hashed name.
|
|
220
|
+
// Otherwise an importer keeps its old hash when a dependency changes,
|
|
221
|
+
// and immutable caching serves a stale importer pointing at a dead file.
|
|
222
|
+
const contents = new Map();
|
|
223
|
+
for (const p of candidates) contents.set(p, await fs.readFile(p));
|
|
224
|
+
const ordered = this.orderByDependencies(candidates, contents);
|
|
225
|
+
|
|
226
|
+
for (const absPath of ordered) {
|
|
227
|
+
let buffer = contents.get(absPath);
|
|
228
|
+
let contentChanged = false;
|
|
229
|
+
|
|
230
|
+
// Rewrite import specifiers of already-fingerprinted dependencies
|
|
231
|
+
// BEFORE hashing, so this file's hash reflects theirs
|
|
232
|
+
if (absPath.endsWith('.js')) {
|
|
233
|
+
const { code, changed } = this.rewriteImportSpecifiers(
|
|
234
|
+
buffer.toString('utf-8'), absPath, manifest
|
|
235
|
+
);
|
|
236
|
+
if (changed) {
|
|
237
|
+
buffer = Buffer.from(code, 'utf-8');
|
|
238
|
+
contentChanged = true;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
|
|
208
242
|
const hash = computeHash(buffer, this.config.hashLength, this.config.algorithm);
|
|
209
243
|
|
|
210
244
|
const dir = path.dirname(absPath);
|
|
@@ -224,6 +258,7 @@ class FingerprintAssets {
|
|
|
224
258
|
await fs.rename(mapAbs, mapNewAbs);
|
|
225
259
|
// Patch sourceMappingURL comment to point at the new map name
|
|
226
260
|
const text = buffer.toString('utf-8');
|
|
261
|
+
contentChanged = false; // superseded by updatedContent below
|
|
227
262
|
updatedContent = text.replace(
|
|
228
263
|
/(sourceMappingURL=)([^\s*'"]+)/,
|
|
229
264
|
(_, prefix) => `${prefix}${mapNewName}`
|
|
@@ -235,18 +270,22 @@ class FingerprintAssets {
|
|
|
235
270
|
if (updatedContent !== null) {
|
|
236
271
|
await fs.writeFile(newAbs, updatedContent, 'utf-8');
|
|
237
272
|
await fs.unlink(absPath);
|
|
273
|
+
} else if (contentChanged) {
|
|
274
|
+
// Import specifiers were rewritten — write the updated content
|
|
275
|
+
await fs.writeFile(newAbs, buffer);
|
|
276
|
+
await fs.unlink(absPath);
|
|
238
277
|
} else {
|
|
239
278
|
await fs.rename(absPath, newAbs);
|
|
240
279
|
}
|
|
241
280
|
|
|
242
281
|
// Build manifest keys as web paths starting with `/`
|
|
243
|
-
const relOriginal = '/' + path.relative(distPath, absPath).split(path.sep).join('/');
|
|
244
|
-
const relNew = '/' + path.relative(distPath, newAbs).split(path.sep).join('/');
|
|
282
|
+
const relOriginal = '/' + path.relative(this.distPath, absPath).split(path.sep).join('/');
|
|
283
|
+
const relNew = '/' + path.relative(this.distPath, newAbs).split(path.sep).join('/');
|
|
245
284
|
manifest[relOriginal] = relNew;
|
|
246
285
|
|
|
247
286
|
if (mapNewName) {
|
|
248
|
-
const relMapOriginal = '/' + path.relative(distPath, mapAbs).split(path.sep).join('/');
|
|
249
|
-
const relMapNew = '/' + path.relative(distPath, path.join(dir, mapNewName)).split(path.sep).join('/');
|
|
287
|
+
const relMapOriginal = '/' + path.relative(this.distPath, mapAbs).split(path.sep).join('/');
|
|
288
|
+
const relMapNew = '/' + path.relative(this.distPath, path.join(dir, mapNewName)).split(path.sep).join('/');
|
|
250
289
|
manifest[relMapOriginal] = relMapNew;
|
|
251
290
|
}
|
|
252
291
|
}
|
|
@@ -254,6 +293,87 @@ class FingerprintAssets {
|
|
|
254
293
|
return manifest;
|
|
255
294
|
}
|
|
256
295
|
|
|
296
|
+
/**
|
|
297
|
+
* Resolve an import specifier to a manifest key (absolute-from-dist web
|
|
298
|
+
* path), or null when it can't be fingerprinted (external URL, bare
|
|
299
|
+
* package specifier).
|
|
300
|
+
* @param {string} spec - Raw specifier from the source
|
|
301
|
+
* @param {string} absPath - File the specifier appears in
|
|
302
|
+
* @returns {string|null}
|
|
303
|
+
*/
|
|
304
|
+
resolveSpecifierKey(spec, absPath) {
|
|
305
|
+
if (isExternalRef(spec)) return null;
|
|
306
|
+
const clean = splitUrlSuffix(spec).path;
|
|
307
|
+
if (clean.startsWith('/')) return clean;
|
|
308
|
+
if (clean.startsWith('./') || clean.startsWith('../')) {
|
|
309
|
+
const dirRel = path.dirname(path.relative(this.distPath, absPath));
|
|
310
|
+
const joined = path.normalize(path.join(dirRel, clean));
|
|
311
|
+
return '/' + joined.split(path.sep).join('/');
|
|
312
|
+
}
|
|
313
|
+
// Bare specifier (npm package / import map) — leave alone
|
|
314
|
+
return null;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Rewrite import/export/dynamic-import specifiers whose targets exist in
|
|
319
|
+
* the manifest. Fingerprinting only changes the basename, so the
|
|
320
|
+
* specifier's directory part is preserved as written.
|
|
321
|
+
* @param {string} code
|
|
322
|
+
* @param {string} absPath - File the code came from
|
|
323
|
+
* @param {Object} manifest
|
|
324
|
+
* @returns {{ code: string, changed: boolean }}
|
|
325
|
+
*/
|
|
326
|
+
rewriteImportSpecifiers(code, absPath, manifest) {
|
|
327
|
+
let changed = false;
|
|
328
|
+
const out = code.replace(SPECIFIER_PATTERN, (match, pre, spec, post) => {
|
|
329
|
+
const key = this.resolveSpecifierKey(spec, absPath);
|
|
330
|
+
if (!key || !manifest[key]) return match;
|
|
331
|
+
const newBase = path.basename(manifest[key]);
|
|
332
|
+
const slashIdx = spec.lastIndexOf('/');
|
|
333
|
+
const newSpec = slashIdx === -1 ? newBase : spec.slice(0, slashIdx + 1) + newBase;
|
|
334
|
+
changed = true;
|
|
335
|
+
return `${pre}${newSpec}${post}`;
|
|
336
|
+
});
|
|
337
|
+
return { code: out, changed };
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/**
|
|
341
|
+
* Order candidates so JS dependencies precede their importers
|
|
342
|
+
* (depth-first topological sort, cycle-tolerant).
|
|
343
|
+
* @param {string[]} candidates - Absolute paths
|
|
344
|
+
* @param {Map<string,Buffer>} contents
|
|
345
|
+
* @returns {string[]}
|
|
346
|
+
*/
|
|
347
|
+
orderByDependencies(candidates, contents) {
|
|
348
|
+
const keyOf = p => '/' + path.relative(this.distPath, p).split(path.sep).join('/');
|
|
349
|
+
const byKey = new Map(candidates.map(p => [keyOf(p), p]));
|
|
350
|
+
|
|
351
|
+
const deps = new Map();
|
|
352
|
+
for (const p of candidates) {
|
|
353
|
+
const set = new Set();
|
|
354
|
+
if (p.endsWith('.js')) {
|
|
355
|
+
const code = contents.get(p).toString('utf-8');
|
|
356
|
+
for (const m of code.matchAll(SPECIFIER_PATTERN)) {
|
|
357
|
+
const key = this.resolveSpecifierKey(m[2], p);
|
|
358
|
+
const target = key && byKey.get(key);
|
|
359
|
+
if (target && target !== p) set.add(target);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
deps.set(p, set);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const ordered = [];
|
|
366
|
+
const state = new Map();
|
|
367
|
+
const visit = (p) => {
|
|
368
|
+
if (state.has(p)) return;
|
|
369
|
+
state.set(p, true);
|
|
370
|
+
for (const d of deps.get(p)) visit(d);
|
|
371
|
+
ordered.push(p);
|
|
372
|
+
};
|
|
373
|
+
for (const p of candidates) visit(p);
|
|
374
|
+
return ordered;
|
|
375
|
+
}
|
|
376
|
+
|
|
257
377
|
|
|
258
378
|
// EXPORT WRAPPER
|
|
259
379
|
// -----------------------------
|
|
@@ -26,9 +26,15 @@ import { distPath, srcPath, formats as formatsConfig } from '../utils/config/con
|
|
|
26
26
|
// DEFINE
|
|
27
27
|
// -----------------------------
|
|
28
28
|
class GenerateFormats {
|
|
29
|
-
constructor({ store, data }) {
|
|
29
|
+
constructor({ store, data, pages, distPath: distPathOverride }) {
|
|
30
30
|
this.store = store;
|
|
31
31
|
this.data = data || {};
|
|
32
|
+
// Optional per-call overrides
|
|
33
|
+
this.distPath = distPathOverride || distPath;
|
|
34
|
+
// Optional shared page snapshot (see scripts/utils/site-pages) —
|
|
35
|
+
// avoids re-walking dist and re-parsing every page. Doms are shared
|
|
36
|
+
// and must be treated as read-only.
|
|
37
|
+
this.pages = pages || null;
|
|
32
38
|
this.config = {
|
|
33
39
|
markdown: true,
|
|
34
40
|
json: true,
|
|
@@ -57,15 +63,19 @@ class GenerateFormats {
|
|
|
57
63
|
this.startLog();
|
|
58
64
|
|
|
59
65
|
try {
|
|
60
|
-
//
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
66
|
+
// Use the shared snapshot when provided; otherwise walk + parse
|
|
67
|
+
let pages = this.pages;
|
|
68
|
+
if (!pages) {
|
|
69
|
+
const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
|
|
70
|
+
const indexFiles = htmlFiles
|
|
71
|
+
.filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
|
|
72
|
+
pages = indexFiles.map(filePath => ({ filePath, urlPath: null, dom: null }));
|
|
73
|
+
}
|
|
74
|
+
pages = pages.filter(p => !p.filePath.includes('/404/'));
|
|
65
75
|
|
|
66
76
|
// Process each page
|
|
67
|
-
for (const
|
|
68
|
-
await this.processPage(
|
|
77
|
+
for (const page of pages) {
|
|
78
|
+
await this.processPage(page);
|
|
69
79
|
}
|
|
70
80
|
|
|
71
81
|
// Generate feed
|
|
@@ -75,15 +85,15 @@ class GenerateFormats {
|
|
|
75
85
|
|
|
76
86
|
// Generate llms.txt
|
|
77
87
|
if (this.config.llmsTxt) {
|
|
78
|
-
await this.generateLlmsTxt(
|
|
88
|
+
await this.generateLlmsTxt(pages);
|
|
79
89
|
}
|
|
80
90
|
|
|
81
91
|
// Generate llms-full.txt (complete documentation in one file)
|
|
82
92
|
if (this.config.llmsFullTxt) {
|
|
83
|
-
await this.generateLlmsFullTxt(
|
|
93
|
+
await this.generateLlmsFullTxt(pages);
|
|
84
94
|
}
|
|
85
95
|
} catch (err) {
|
|
86
|
-
Util.
|
|
96
|
+
Util.customWarn(err, 'generate-formats.js (formats skipped)');
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
// END LOGGING
|
|
@@ -92,13 +102,13 @@ class GenerateFormats {
|
|
|
92
102
|
|
|
93
103
|
/**
|
|
94
104
|
* Process a single page: generate markdown copy and JSON metadata
|
|
95
|
-
* @param {
|
|
105
|
+
* @param {Object} page - { filePath, urlPath?, dom? } snapshot entry
|
|
96
106
|
*/
|
|
97
|
-
async processPage(
|
|
107
|
+
async processPage(page) {
|
|
98
108
|
// Get the URL path from the dist file path
|
|
99
109
|
// e.g., dist/blog/my-post/index.html → /blog/my-post/
|
|
100
|
-
const
|
|
101
|
-
|
|
110
|
+
const urlPath = page.urlPath
|
|
111
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
102
112
|
|
|
103
113
|
// Try to find corresponding markdown source
|
|
104
114
|
const mdSourcePath = this.findMarkdownSource(urlPath);
|
|
@@ -110,7 +120,7 @@ class GenerateFormats {
|
|
|
110
120
|
|
|
111
121
|
// Generate JSON metadata
|
|
112
122
|
if (this.config.json) {
|
|
113
|
-
await this.generateJsonMeta(
|
|
123
|
+
await this.generateJsonMeta(page, urlPath, mdSourcePath);
|
|
114
124
|
}
|
|
115
125
|
}
|
|
116
126
|
|
|
@@ -146,14 +156,14 @@ class GenerateFormats {
|
|
|
146
156
|
const content = await fs.readFile(mdSourcePath, 'utf-8');
|
|
147
157
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
|
|
148
158
|
const outputPath = trimmedPath
|
|
149
|
-
? path.resolve(`${distPath}/md/${trimmedPath}/index.md`)
|
|
150
|
-
: path.resolve(`${distPath}/md/index.md`);
|
|
159
|
+
? path.resolve(`${this.distPath}/md/${trimmedPath}/index.md`)
|
|
160
|
+
: path.resolve(`${this.distPath}/md/index.md`);
|
|
151
161
|
|
|
152
162
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
153
163
|
await fs.writeFile(outputPath, content, 'utf-8');
|
|
154
164
|
this.stats.markdown++;
|
|
155
165
|
} catch (err) {
|
|
156
|
-
Util.
|
|
166
|
+
Util.customWarn(err, `generate-formats.js: markdown for ${urlPath} (skipped)`);
|
|
157
167
|
}
|
|
158
168
|
}
|
|
159
169
|
|
|
@@ -163,10 +173,10 @@ class GenerateFormats {
|
|
|
163
173
|
* @param {string} urlPath - URL path
|
|
164
174
|
* @param {string|null} mdSourcePath - Path to markdown source if exists
|
|
165
175
|
*/
|
|
166
|
-
async generateJsonMeta(
|
|
176
|
+
async generateJsonMeta(page, urlPath, mdSourcePath) {
|
|
167
177
|
try {
|
|
168
|
-
const
|
|
169
|
-
|
|
178
|
+
const dom = page.dom
|
|
179
|
+
|| Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
|
|
170
180
|
const document = dom.window.document;
|
|
171
181
|
|
|
172
182
|
// Extract metadata from the HTML
|
|
@@ -198,13 +208,13 @@ class GenerateFormats {
|
|
|
198
208
|
|
|
199
209
|
// Write JSON
|
|
200
210
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '') || 'index';
|
|
201
|
-
const outputPath = path.resolve(`${distPath}/api/${trimmedPath}.json`);
|
|
211
|
+
const outputPath = path.resolve(`${this.distPath}/api/${trimmedPath}.json`);
|
|
202
212
|
|
|
203
213
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
204
214
|
await fs.writeFile(outputPath, JSON.stringify(meta, null, 2), 'utf-8');
|
|
205
215
|
this.stats.json++;
|
|
206
216
|
} catch (err) {
|
|
207
|
-
Util.
|
|
217
|
+
Util.customWarn(err, `generate-formats.js: json for ${urlPath} (skipped)`);
|
|
208
218
|
}
|
|
209
219
|
}
|
|
210
220
|
|
|
@@ -273,7 +283,7 @@ class GenerateFormats {
|
|
|
273
283
|
xml += `</feed>
|
|
274
284
|
`;
|
|
275
285
|
|
|
276
|
-
await fs.writeFile(path.resolve(`${distPath}/feed.xml`), xml, 'utf-8');
|
|
286
|
+
await fs.writeFile(path.resolve(`${this.distPath}/feed.xml`), xml, 'utf-8');
|
|
277
287
|
this.stats.feed = true;
|
|
278
288
|
}
|
|
279
289
|
|
|
@@ -283,11 +293,11 @@ class GenerateFormats {
|
|
|
283
293
|
* @param {string[]} indexFiles - All index.html file paths
|
|
284
294
|
* @returns {Array<{title: string, description: string, url: string, mdUrl: string|null, section: string|null}>}
|
|
285
295
|
*/
|
|
286
|
-
async collectPageMeta(
|
|
296
|
+
async collectPageMeta(pageEntries) {
|
|
287
297
|
const pages = [];
|
|
288
|
-
for (const
|
|
289
|
-
const
|
|
290
|
-
|
|
298
|
+
for (const page of pageEntries) {
|
|
299
|
+
const urlPath = page.urlPath
|
|
300
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
291
301
|
const mdSource = this.findMarkdownSource(urlPath);
|
|
292
302
|
const trimmedPath = urlPath.replace(/^\//, '').replace(/\/$/, '');
|
|
293
303
|
|
|
@@ -296,8 +306,8 @@ class GenerateFormats {
|
|
|
296
306
|
let description = '';
|
|
297
307
|
let section = null;
|
|
298
308
|
try {
|
|
299
|
-
const
|
|
300
|
-
|
|
309
|
+
const dom = page.dom
|
|
310
|
+
|| Util.jsdom.dom({ src: await fs.readFile(page.filePath, 'utf-8') });
|
|
301
311
|
const doc = dom.window.document;
|
|
302
312
|
const h1 = doc.querySelector('h1')?.textContent;
|
|
303
313
|
const titleTag = doc.querySelector('title')?.textContent;
|
|
@@ -328,7 +338,7 @@ class GenerateFormats {
|
|
|
328
338
|
* Enhanced with descriptions, section grouping, and AI instructions.
|
|
329
339
|
* @param {string[]} indexFiles - All index.html file paths
|
|
330
340
|
*/
|
|
331
|
-
async generateLlmsTxt(
|
|
341
|
+
async generateLlmsTxt(pageEntries) {
|
|
332
342
|
const siteTitle = this.data.siteTitle || 'Site';
|
|
333
343
|
const siteDescription = this.data.siteDescription || '';
|
|
334
344
|
const siteUrl = this.data.siteUrl || '';
|
|
@@ -343,7 +353,7 @@ class GenerateFormats {
|
|
|
343
353
|
}
|
|
344
354
|
|
|
345
355
|
// Collect page metadata
|
|
346
|
-
const pages = await this.collectPageMeta(
|
|
356
|
+
const pages = await this.collectPageMeta(pageEntries);
|
|
347
357
|
const mdPages = pages.filter(p => p.mdUrl);
|
|
348
358
|
|
|
349
359
|
if (mdPages.length) {
|
|
@@ -393,7 +403,7 @@ class GenerateFormats {
|
|
|
393
403
|
}
|
|
394
404
|
}
|
|
395
405
|
|
|
396
|
-
await fs.writeFile(path.resolve(`${distPath}/llms.txt`), content, 'utf-8');
|
|
406
|
+
await fs.writeFile(path.resolve(`${this.distPath}/llms.txt`), content, 'utf-8');
|
|
397
407
|
this.stats.llmsTxt = true;
|
|
398
408
|
}
|
|
399
409
|
|
|
@@ -402,7 +412,7 @@ class GenerateFormats {
|
|
|
402
412
|
* Ideal for bulk ingestion by IDE tools and AI systems.
|
|
403
413
|
* @param {string[]} indexFiles - All index.html file paths
|
|
404
414
|
*/
|
|
405
|
-
async generateLlmsFullTxt(
|
|
415
|
+
async generateLlmsFullTxt(pageEntries) {
|
|
406
416
|
const siteTitle = this.data.siteTitle || 'Site';
|
|
407
417
|
const siteDescription = this.data.siteDescription || '';
|
|
408
418
|
const llmsInstructions = this.data.llmsInstructions || '';
|
|
@@ -416,9 +426,9 @@ class GenerateFormats {
|
|
|
416
426
|
}
|
|
417
427
|
|
|
418
428
|
// Collect pages and read their markdown content
|
|
419
|
-
for (const
|
|
420
|
-
const
|
|
421
|
-
|
|
429
|
+
for (const page of pageEntries) {
|
|
430
|
+
const urlPath = page.urlPath
|
|
431
|
+
|| page.filePath.replace(this.distPath, '').replace(/\/index\.html$/, '/').replace(/^$/, '/');
|
|
422
432
|
const mdSource = this.findMarkdownSource(urlPath);
|
|
423
433
|
if (!mdSource) continue;
|
|
424
434
|
|
|
@@ -439,7 +449,7 @@ class GenerateFormats {
|
|
|
439
449
|
} catch { /* skip unreadable files */ }
|
|
440
450
|
}
|
|
441
451
|
|
|
442
|
-
await fs.writeFile(path.resolve(`${distPath}/llms-full.txt`), content, 'utf-8');
|
|
452
|
+
await fs.writeFile(path.resolve(`${this.distPath}/llms-full.txt`), content, 'utf-8');
|
|
443
453
|
this.stats.llmsFullTxt = true;
|
|
444
454
|
}
|
|
445
455
|
|
|
@@ -22,8 +22,13 @@ import { distPath, fragments as fragmentsConfig } from '../utils/config/config.j
|
|
|
22
22
|
// DEFINE
|
|
23
23
|
// -----------------------------
|
|
24
24
|
class GenerateFragments {
|
|
25
|
-
constructor() {
|
|
25
|
+
constructor(opts = {}) {
|
|
26
26
|
this.total = 0;
|
|
27
|
+
// Optional per-call overrides
|
|
28
|
+
this.distPath = opts.distPath || distPath;
|
|
29
|
+
// Optional shared page snapshot (see scripts/utils/site-pages) —
|
|
30
|
+
// avoids re-walking dist and re-parsing every page
|
|
31
|
+
this.pages = opts.pages || null;
|
|
27
32
|
|
|
28
33
|
// Merge defaults with user config
|
|
29
34
|
this.config = {
|
|
@@ -50,15 +55,22 @@ class GenerateFragments {
|
|
|
50
55
|
this.startLog();
|
|
51
56
|
|
|
52
57
|
try {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
58
|
+
if (this.pages) {
|
|
59
|
+
// Shared snapshot: pages are already read and parsed
|
|
60
|
+
for (const page of this.pages) {
|
|
61
|
+
await this.generateFragment(page.filePath, page.dom);
|
|
62
|
+
}
|
|
63
|
+
} else {
|
|
64
|
+
// Standalone: walk and parse ourselves
|
|
65
|
+
const htmlFiles = Util.getPaths(this.distPath, this.distPath, []);
|
|
66
|
+
const indexFiles = htmlFiles.filter(f => f.endsWith('/index.html') || f === `${this.distPath}/index.html`);
|
|
67
|
+
|
|
68
|
+
for (const filePath of indexFiles) {
|
|
69
|
+
await this.generateFragment(filePath);
|
|
70
|
+
}
|
|
59
71
|
}
|
|
60
72
|
} catch (err) {
|
|
61
|
-
Util.
|
|
73
|
+
Util.customWarn(err, 'generate-fragments.js (fragments skipped)');
|
|
62
74
|
}
|
|
63
75
|
|
|
64
76
|
// END LOGGING
|
|
@@ -68,13 +80,11 @@ class GenerateFragments {
|
|
|
68
80
|
/**
|
|
69
81
|
* Generate a fragment file for a single HTML page
|
|
70
82
|
* @param {string} filePath - Path to the index.html file
|
|
83
|
+
* @param {Object} [sharedDom] - Pre-parsed document from the shared snapshot (read-only)
|
|
71
84
|
*/
|
|
72
|
-
async generateFragment(filePath) {
|
|
85
|
+
async generateFragment(filePath, sharedDom) {
|
|
73
86
|
try {
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
// Parse with JSDOM
|
|
77
|
-
const dom = Util.jsdom.dom({ src });
|
|
87
|
+
const dom = sharedDom || Util.jsdom.dom({ src: await fs.readFile(filePath, 'utf-8') });
|
|
78
88
|
const document = dom.window.document;
|
|
79
89
|
|
|
80
90
|
// Find the target element
|
|
@@ -91,7 +101,7 @@ class GenerateFragments {
|
|
|
91
101
|
|
|
92
102
|
this.total++;
|
|
93
103
|
} catch (err) {
|
|
94
|
-
Util.
|
|
104
|
+
Util.customWarn(err, `generate-fragments.js: ${filePath} (skipped)`);
|
|
95
105
|
}
|
|
96
106
|
}
|
|
97
107
|
|
|
@@ -8,7 +8,7 @@ import Logger from '../utils/logger/logger.js';
|
|
|
8
8
|
import Util from '../utils/util/util.js';
|
|
9
9
|
|
|
10
10
|
// CONFIG
|
|
11
|
-
import { distPath, sitemap, srcPath } from '../utils/config/config.js';
|
|
11
|
+
import { distPath, fragments, sitemap, srcPath } from '../utils/config/config.js';
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
// DEFINE
|
|
@@ -18,7 +18,14 @@ import { distPath, sitemap, srcPath } from '../utils/config/config.js';
|
|
|
18
18
|
* Is one of the last build processes since we want to get all new, dynamically-generated pages and their final path locations.
|
|
19
19
|
*/
|
|
20
20
|
class GenerateSitemap {
|
|
21
|
-
constructor() {
|
|
21
|
+
constructor(opts = {}) {
|
|
22
|
+
// Optional per-call overrides (used by tests and custom pipelines)
|
|
23
|
+
this.distPath = opts.distPath || distPath;
|
|
24
|
+
this.sitemap = opts.sitemap || sitemap;
|
|
25
|
+
// SPA fragment filename — fragment files are content duplicates of
|
|
26
|
+
// their parent page and must never be listed as URLs
|
|
27
|
+
this.fragmentsFilename = (fragments && fragments.filename) || '_fragment.html';
|
|
28
|
+
|
|
22
29
|
// Store total # of file entries generated
|
|
23
30
|
this.total = 0;
|
|
24
31
|
|
|
@@ -31,7 +38,7 @@ class GenerateSitemap {
|
|
|
31
38
|
// Note: `process.env.DEV_CHANGED_PAGE` is defined in `browserSync.watch()` in dev.js
|
|
32
39
|
async init() {
|
|
33
40
|
// Early Exit: No user-defined site domain in `/config/main.js`
|
|
34
|
-
if (!sitemap || !sitemap.url || !sitemap.url.length) return;
|
|
41
|
+
if (!this.sitemap || !this.sitemap.url || !this.sitemap.url.length) return;
|
|
35
42
|
|
|
36
43
|
// ADD TERMINAL SECTION HEADING
|
|
37
44
|
Logger.persist.header(`\nCreate /sitemap.xml`);
|
|
@@ -40,20 +47,19 @@ class GenerateSitemap {
|
|
|
40
47
|
this.startLog();
|
|
41
48
|
|
|
42
49
|
try {
|
|
43
|
-
// Get directory and .html file paths from `/dist`
|
|
44
|
-
let files = await fs.readdir(distPath);
|
|
45
|
-
this.total = files.length;
|
|
46
50
|
// Format file paths for XML
|
|
47
|
-
files = this.formatFilesForXML(
|
|
51
|
+
let files = this.formatFilesForXML();
|
|
48
52
|
// Filter out unwanted entry paths
|
|
49
53
|
files = this.excludeFiles(files);
|
|
54
|
+
// Store total # of actual sitemap entries (for logging)
|
|
55
|
+
this.total = files.length;
|
|
50
56
|
// Build XML source
|
|
51
57
|
let xmlSrc = this.buildXML(files);
|
|
52
58
|
// Create `sitemap.xml` and write source to it
|
|
53
|
-
await fs.writeFile(`${distPath}/sitemap.xml`, xmlSrc);
|
|
59
|
+
await fs.writeFile(`${this.distPath}/sitemap.xml`, xmlSrc);
|
|
54
60
|
}
|
|
55
61
|
catch (e) {
|
|
56
|
-
Util.
|
|
62
|
+
Util.customWarn(e, 'Generate sitemap.xml (skipped)');
|
|
57
63
|
Logger.error(`Requires /${distPath} directory - please run the build process first`);
|
|
58
64
|
}
|
|
59
65
|
|
|
@@ -73,14 +79,14 @@ class GenerateSitemap {
|
|
|
73
79
|
let date = this.formattedDate();
|
|
74
80
|
let xml = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
|
|
75
81
|
// If `sitemap.url` has a trailing /, remove it to avoid potential double slashes (ex: https://domain.com//path/to/file)
|
|
76
|
-
const domain = sitemap.url[sitemap.url.length - 1] === '/' ? sitemap.url.slice(0, -1) : sitemap.url;
|
|
82
|
+
const domain = this.sitemap.url[this.sitemap.url.length - 1] === '/' ? this.sitemap.url.slice(0, -1) : this.sitemap.url;
|
|
77
83
|
files.forEach(f => {
|
|
78
84
|
// Set homepage to higher priority
|
|
79
85
|
const priority = f === '' ? '1.0' : '0.9';
|
|
80
86
|
// Add entry
|
|
81
87
|
xml += `
|
|
82
88
|
<url>
|
|
83
|
-
<loc>${domain}${f}</loc>
|
|
89
|
+
<loc>${this.escapeXml(`${domain}${f}`)}</loc>
|
|
84
90
|
<lastmod>${date}</lastmod>
|
|
85
91
|
<changefreq>monthly</changefreq>
|
|
86
92
|
<priority>${priority}</priority>
|
|
@@ -103,7 +109,7 @@ class GenerateSitemap {
|
|
|
103
109
|
// Default path regexes: Any `/assets`, `/includes`, and root-level `404.html` files
|
|
104
110
|
const defaultExcludePaths = [/^\/assets/, /^\/includes/, /^\/404.html/];
|
|
105
111
|
// User defined regexes from `/config/main.js`
|
|
106
|
-
const userExcludePaths = sitemap && sitemap.excludePaths || [];
|
|
112
|
+
const userExcludePaths = this.sitemap && this.sitemap.excludePaths || [];
|
|
107
113
|
// Combine the default and user regexes
|
|
108
114
|
const excludePaths = [...defaultExcludePaths, ...userExcludePaths];
|
|
109
115
|
// Filter out unwanted paths
|
|
@@ -125,16 +131,18 @@ class GenerateSitemap {
|
|
|
125
131
|
* @param {Array} files - The array of site file paths
|
|
126
132
|
* @returns {Array}
|
|
127
133
|
*/
|
|
128
|
-
formatFilesForXML(
|
|
134
|
+
formatFilesForXML() {
|
|
129
135
|
// Allowed page extensions
|
|
130
136
|
const allowedExt = ['html'];
|
|
131
137
|
const excludedPaths = [];
|
|
132
138
|
// Get files in `/dist`
|
|
133
|
-
files = Util.getPaths(distPath, distPath, excludedPaths);
|
|
139
|
+
let files = Util.getPaths(this.distPath, this.distPath, excludedPaths);
|
|
134
140
|
// Get only the allowed files by extension (.css, .html)
|
|
135
141
|
files = files.filter(fileName => Util.isExtension(fileName, allowedExt));
|
|
142
|
+
// Skip SPA fragment files (content duplicates of their parent page)
|
|
143
|
+
files = files.filter(fileName => !fileName.endsWith(`/${this.fragmentsFilename}`));
|
|
136
144
|
// Format for xml: Remove `/dist`
|
|
137
|
-
files = files.map(fileName => fileName.split(distPath)[1]);
|
|
145
|
+
files = files.map(fileName => fileName.split(this.distPath)[1]);
|
|
138
146
|
// Format for xml: Remove `filename`
|
|
139
147
|
files = files.map(fileName => this.formatDirPath(fileName));
|
|
140
148
|
// Alphabetize files
|
|
@@ -146,6 +154,21 @@ class GenerateSitemap {
|
|
|
146
154
|
// HELPER METHODS
|
|
147
155
|
// -----------------------------
|
|
148
156
|
|
|
157
|
+
/**
|
|
158
|
+
* @description Escape XML-special characters for <loc> values
|
|
159
|
+
* @param {String} str - Raw URL string
|
|
160
|
+
* @returns {String}
|
|
161
|
+
* @private
|
|
162
|
+
*/
|
|
163
|
+
escapeXml(str) {
|
|
164
|
+
return str
|
|
165
|
+
.replace(/&/g, '&')
|
|
166
|
+
.replace(/</g, '<')
|
|
167
|
+
.replace(/>/g, '>')
|
|
168
|
+
.replace(/"/g, '"')
|
|
169
|
+
.replace(/'/g, ''');
|
|
170
|
+
}
|
|
171
|
+
|
|
149
172
|
/**
|
|
150
173
|
* @description Format current date in `yyyy-mm-dd` format
|
|
151
174
|
* @returns {String}
|