@eox/pages-theme-eox 2.0.0-alpha.5 → 2.0.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitlab-ci.yml +4 -1
- package/README.md +93 -4
- package/cypress/support/mocks/authors.data.js +14 -0
- package/cypress/support/mocks/helpers.js +40 -1
- package/cypress.config.js +9 -0
- package/package.json +5 -2
- package/src/components/BlogPost.vue +13 -1
- package/src/components/FeatureSection.cy.js +16 -0
- package/src/components/Footer.cy.js +12 -0
- package/src/components/Footer.vue +7 -0
- package/src/components/MobileNavDropdown.vue +7 -0
- package/src/components/NavDropdown.vue +7 -0
- package/src/components/TestimonialsSection.cy.js +2 -5
- package/src/helpers.js +169 -32
- package/src/helpers.test.js +153 -0
- package/src/llms/index.js +64 -0
- package/src/llms/llms.test.js +980 -0
- package/src/llms/storiesPlugin.js +771 -0
- package/src/pages/author/[author].paths.js +2 -2
- package/src/posts.data.js +39 -1
- package/src/style.css +140 -0
- package/src/vitepressConfig.mjs +7 -1
|
@@ -0,0 +1,771 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import pc from "picocolors";
|
|
4
|
+
import { createLogger } from "vite";
|
|
5
|
+
|
|
6
|
+
const prefix = pc.blue("llmstxt (stories)") + pc.dim(" »");
|
|
7
|
+
let logger = createLogger("info", { prefix });
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Creates the stories build-time Vite plugin.
|
|
11
|
+
* @param {object} params
|
|
12
|
+
* @param {object | boolean | string} [params.storiesOption]
|
|
13
|
+
* @param {object} [params.brandConfig]
|
|
14
|
+
* @param {string | null} [params.domain]
|
|
15
|
+
* @returns {import("vite").Plugin}
|
|
16
|
+
*/
|
|
17
|
+
export function createStoriesPlugin({
|
|
18
|
+
storiesOption,
|
|
19
|
+
brandConfig = {},
|
|
20
|
+
domain = null,
|
|
21
|
+
}) {
|
|
22
|
+
let outDir = "dist";
|
|
23
|
+
let base = "/";
|
|
24
|
+
let srcDir = process.cwd();
|
|
25
|
+
let isSsrBuild = false;
|
|
26
|
+
|
|
27
|
+
const config =
|
|
28
|
+
typeof storiesOption === "object" && storiesOption !== null
|
|
29
|
+
? storiesOption
|
|
30
|
+
: typeof storiesOption === "string"
|
|
31
|
+
? { manifest: storiesOption }
|
|
32
|
+
: {};
|
|
33
|
+
|
|
34
|
+
const heading = config.heading || "Stories";
|
|
35
|
+
const concurrency = config.concurrency || 8;
|
|
36
|
+
|
|
37
|
+
return {
|
|
38
|
+
name: "@eox/pages-theme-eox:llms-stories",
|
|
39
|
+
enforce: "post",
|
|
40
|
+
apply: "build",
|
|
41
|
+
configResolved(resolvedConfig) {
|
|
42
|
+
logger = createLogger(resolvedConfig.logLevel, {
|
|
43
|
+
prefix,
|
|
44
|
+
allowClearScreen: resolvedConfig.clearScreen,
|
|
45
|
+
});
|
|
46
|
+
outDir =
|
|
47
|
+
resolvedConfig.vitepress?.outDir ||
|
|
48
|
+
(resolvedConfig.build?.outDir
|
|
49
|
+
? path.resolve(
|
|
50
|
+
resolvedConfig.root || process.cwd(),
|
|
51
|
+
resolvedConfig.build.outDir,
|
|
52
|
+
)
|
|
53
|
+
: path.resolve(resolvedConfig.root || process.cwd(), "dist"));
|
|
54
|
+
base = resolvedConfig.base || resolvedConfig.vitepress?.site?.base || "/";
|
|
55
|
+
srcDir =
|
|
56
|
+
resolvedConfig.vitepress?.srcDir ||
|
|
57
|
+
resolvedConfig.root ||
|
|
58
|
+
process.cwd();
|
|
59
|
+
isSsrBuild = Boolean(resolvedConfig.build?.ssr);
|
|
60
|
+
},
|
|
61
|
+
async closeBundle() {
|
|
62
|
+
if (isSsrBuild) return;
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const resolved = await resolveAndFetchManifest({
|
|
66
|
+
explicitUrl: config.manifest,
|
|
67
|
+
brandStories: brandConfig.stories,
|
|
68
|
+
srcDir,
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
if (!resolved) return;
|
|
72
|
+
const { manifestUrl, manifest, sourceMarkdownPath } = resolved;
|
|
73
|
+
|
|
74
|
+
const manifestBaseUrl = manifestUrl.substring(
|
|
75
|
+
0,
|
|
76
|
+
manifestUrl.lastIndexOf("/"),
|
|
77
|
+
);
|
|
78
|
+
const sourceRelPath = sourceMarkdownPath
|
|
79
|
+
? path.relative(srcDir, sourceMarkdownPath)
|
|
80
|
+
: null;
|
|
81
|
+
const routing = resolveStoryRouting({
|
|
82
|
+
sourceRelPath,
|
|
83
|
+
configuredPath: config.storyPath || config.path,
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
const stories = await fetchStories({
|
|
87
|
+
manifest,
|
|
88
|
+
manifestBaseUrl,
|
|
89
|
+
base,
|
|
90
|
+
domain,
|
|
91
|
+
routing,
|
|
92
|
+
concurrency,
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
if (stories.length === 0) return;
|
|
96
|
+
|
|
97
|
+
if (routing.isDynamic) {
|
|
98
|
+
await writeIndividualStoryFiles({
|
|
99
|
+
outDir,
|
|
100
|
+
routeDir: routing.routeDir,
|
|
101
|
+
stories,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
await appendToLlmsTxt({ outDir, heading, stories });
|
|
106
|
+
await appendToLlmsFullTxt({ outDir, heading, stories });
|
|
107
|
+
await appendToStoryRoutePage({
|
|
108
|
+
outDir,
|
|
109
|
+
heading,
|
|
110
|
+
stories,
|
|
111
|
+
sourceMarkdownPath,
|
|
112
|
+
srcDir,
|
|
113
|
+
routing,
|
|
114
|
+
});
|
|
115
|
+
} catch (err) {
|
|
116
|
+
logger.warn(`Failed during stories processing: ${err?.message || err}`);
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Converts a relative markdown file path to a route URL path.
|
|
124
|
+
* @param {string | null | undefined} relPath
|
|
125
|
+
* @returns {string}
|
|
126
|
+
*/
|
|
127
|
+
export function routeFromRelativePath(relPath) {
|
|
128
|
+
if (!relPath || typeof relPath !== "string") return "/story";
|
|
129
|
+
let clean = relPath.replace(/\\/g, "/").replace(/\.(md|html)$/, "");
|
|
130
|
+
if (clean === "index" || clean === "") return "/";
|
|
131
|
+
if (clean.endsWith("/index")) clean = clean.slice(0, -"/index".length);
|
|
132
|
+
return clean.startsWith("/") ? clean : `/${clean}`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const escapeRegex = (str) => str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Strips common markdown and dynamic route script extensions.
|
|
139
|
+
*/
|
|
140
|
+
const ROUTE_EXT_REGEX = /\.(md|html|paths\.[cm]?[jt]s)$/;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Resolves routing information for stories based on source file and configuration.
|
|
144
|
+
* Detects whether the stories are hosted on a dynamic route (e.g. narratives/[narrative].md or [narrative].paths.js)
|
|
145
|
+
* or a single page (e.g. story.md).
|
|
146
|
+
* @param {object} [params]
|
|
147
|
+
* @param {string | null} [params.sourceRelPath]
|
|
148
|
+
* @param {string} [params.configuredPath]
|
|
149
|
+
* @returns {{ isDynamic: boolean, routeDir: string, routePath: string }}
|
|
150
|
+
*/
|
|
151
|
+
export function resolveStoryRouting({
|
|
152
|
+
sourceRelPath = null,
|
|
153
|
+
configuredPath = "",
|
|
154
|
+
} = {}) {
|
|
155
|
+
if (configuredPath) {
|
|
156
|
+
const cleanConfigured = configuredPath
|
|
157
|
+
.replace(/\\/g, "/")
|
|
158
|
+
.replace(ROUTE_EXT_REGEX, "");
|
|
159
|
+
const isDynamic = cleanConfigured.includes("[");
|
|
160
|
+
if (isDynamic) {
|
|
161
|
+
const parts = cleanConfigured.split("/");
|
|
162
|
+
if (parts[parts.length - 1].startsWith("[")) {
|
|
163
|
+
parts.pop();
|
|
164
|
+
}
|
|
165
|
+
const routeDir = parts.join("/").replace(/^\/+|\/+$/g, "");
|
|
166
|
+
const routePath = routeDir ? `/${routeDir}` : "/";
|
|
167
|
+
return { isDynamic, routeDir, routePath };
|
|
168
|
+
}
|
|
169
|
+
const routePath = routeFromRelativePath(cleanConfigured);
|
|
170
|
+
const routeDir = routePath.replace(/^\/+|\/+$/g, "");
|
|
171
|
+
return { isDynamic: false, routeDir, routePath };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (sourceRelPath) {
|
|
175
|
+
const cleanSource = sourceRelPath
|
|
176
|
+
.replace(/\\/g, "/")
|
|
177
|
+
.replace(ROUTE_EXT_REGEX, "");
|
|
178
|
+
if (/(?:^|\/)\[.+?\]/.test(cleanSource)) {
|
|
179
|
+
const parts = cleanSource.split("/");
|
|
180
|
+
parts.pop();
|
|
181
|
+
const routeDir = parts.join("/").replace(/^\/+|\/+$/g, "");
|
|
182
|
+
return {
|
|
183
|
+
isDynamic: true,
|
|
184
|
+
routeDir,
|
|
185
|
+
routePath: routeDir ? `/${routeDir}` : "/",
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const routePath = routeFromRelativePath(sourceRelPath);
|
|
189
|
+
const routeDir = routePath.replace(/^\/+|\/+$/g, "");
|
|
190
|
+
return { isDynamic: false, routeDir, routePath };
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
return { isDynamic: false, routeDir: "story", routePath: "/story" };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Builds a story URL.
|
|
198
|
+
* Dynamic routes produce clean path URLs: /{base}/{routeDir}/{storyId}{extension}
|
|
199
|
+
* Single-page routes produce query param URLs: /{base}/{storyPath}?id={storyId}
|
|
200
|
+
* @param {object} params
|
|
201
|
+
* @param {string | null} [params.domain]
|
|
202
|
+
* @param {string} [params.base]
|
|
203
|
+
* @param {string} [params.routeDir]
|
|
204
|
+
* @param {string} [params.storyPath]
|
|
205
|
+
* @param {string} [params.storyId]
|
|
206
|
+
* @param {boolean} [params.isDynamic]
|
|
207
|
+
* @param {string} [params.extension]
|
|
208
|
+
* @returns {string}
|
|
209
|
+
*/
|
|
210
|
+
export function buildStoryUrl({
|
|
211
|
+
domain = null,
|
|
212
|
+
base = "/",
|
|
213
|
+
routeDir = "story",
|
|
214
|
+
storyPath = "/story",
|
|
215
|
+
storyId = "",
|
|
216
|
+
isDynamic = false,
|
|
217
|
+
extension = "",
|
|
218
|
+
}) {
|
|
219
|
+
const baseSegment = base === "/" ? "" : base.replace(/^\/+|\/+$/g, "");
|
|
220
|
+
|
|
221
|
+
if (isDynamic) {
|
|
222
|
+
const dirSegment = routeDir ? routeDir.replace(/^\/+|\/+$/g, "") : "";
|
|
223
|
+
const idSegment = storyId ? `${storyId}${extension}` : "";
|
|
224
|
+
const segments = [baseSegment, dirSegment, idSegment].filter(Boolean);
|
|
225
|
+
const pathWithBase = `/${segments.join("/")}`;
|
|
226
|
+
return domain ? `${domain}${pathWithBase}` : pathWithBase;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
const pathSegment = storyPath ? storyPath.replace(/^\/+|\/+$/g, "") : "";
|
|
230
|
+
const segments = [baseSegment, pathSegment].filter(Boolean);
|
|
231
|
+
const pathWithBase = `/${segments.join("/")}`;
|
|
232
|
+
const query = storyId ? `?id=${storyId}` : "";
|
|
233
|
+
return domain
|
|
234
|
+
? `${domain}${pathWithBase}${query}`
|
|
235
|
+
: `${pathWithBase}${query}`;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Writes individual sanitized markdown files for dynamic route stories.
|
|
240
|
+
* E.g., dist/narratives/austrian-air-source-heating-potential.md
|
|
241
|
+
* @param {object} params
|
|
242
|
+
* @param {string} params.outDir
|
|
243
|
+
* @param {string} params.routeDir
|
|
244
|
+
* @param {any[]} params.stories
|
|
245
|
+
*/
|
|
246
|
+
export async function writeIndividualStoryFiles({ outDir, routeDir, stories }) {
|
|
247
|
+
const resolvedOutDir = path.resolve(outDir);
|
|
248
|
+
const cleanDir = routeDir
|
|
249
|
+
? routeDir.replace(/\\/g, "/").replace(/^\/+|\/+$/g, "")
|
|
250
|
+
: "";
|
|
251
|
+
const targetDir = cleanDir
|
|
252
|
+
? path.resolve(resolvedOutDir, cleanDir)
|
|
253
|
+
: resolvedOutDir;
|
|
254
|
+
|
|
255
|
+
if (
|
|
256
|
+
targetDir !== resolvedOutDir &&
|
|
257
|
+
!targetDir.startsWith(resolvedOutDir + path.sep)
|
|
258
|
+
) {
|
|
259
|
+
logger.warn(
|
|
260
|
+
`Security warning: targetDir "${targetDir}" is outside outDir "${resolvedOutDir}"`,
|
|
261
|
+
);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
await fs.mkdir(targetDir, { recursive: true });
|
|
266
|
+
|
|
267
|
+
for (const story of stories) {
|
|
268
|
+
if (!story.cleanedBody || !story.storyId) continue;
|
|
269
|
+
try {
|
|
270
|
+
const safeId = path.basename(story.storyId);
|
|
271
|
+
if (!safeId || safeId === "." || safeId === "..") continue;
|
|
272
|
+
const filePath = path.resolve(targetDir, `${safeId}.md`);
|
|
273
|
+
if (!filePath.startsWith(resolvedOutDir + path.sep)) {
|
|
274
|
+
logger.warn(
|
|
275
|
+
`Security warning: skipping file path outside outDir: ${filePath}`,
|
|
276
|
+
);
|
|
277
|
+
continue;
|
|
278
|
+
}
|
|
279
|
+
const content = `# ${story.title}\n\n${story.cleanedBody}\n`;
|
|
280
|
+
await fs.writeFile(filePath, content, "utf8");
|
|
281
|
+
} catch (err) {
|
|
282
|
+
logger.warn(
|
|
283
|
+
`Failed to write story file for ${story.storyId}: ${err?.message || err}`,
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Normalizes text by trimming and collapsing contiguous whitespace.
|
|
291
|
+
* @param {string | undefined | null} str
|
|
292
|
+
* @returns {string}
|
|
293
|
+
*/
|
|
294
|
+
export function normalizeWhitespace(str) {
|
|
295
|
+
if (!str || typeof str !== "string") return "";
|
|
296
|
+
return str.replace(/\s+/g, " ").trim();
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* Derives story id from file path matching upstream behavior:
|
|
301
|
+
* last path segment, truncated at the first dot.
|
|
302
|
+
* @param {string} filePath
|
|
303
|
+
* @returns {string}
|
|
304
|
+
*/
|
|
305
|
+
export function deriveStoryId(filePath) {
|
|
306
|
+
if (!filePath || typeof filePath !== "string") return "";
|
|
307
|
+
const filename = filePath.split("/").pop() || "";
|
|
308
|
+
return filename.split(".")[0];
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Cleans story markdown by stripping metadata, comments, and normalizing headings.
|
|
313
|
+
* @param {string} content
|
|
314
|
+
* @param {string} title
|
|
315
|
+
* @param {string} [manifestBaseUrl]
|
|
316
|
+
* @returns {string}
|
|
317
|
+
*/
|
|
318
|
+
export function cleanStory(content, title, manifestBaseUrl = "") {
|
|
319
|
+
if (!content || typeof content !== "string") return "";
|
|
320
|
+
|
|
321
|
+
let cleaned = content
|
|
322
|
+
.replace(/^---[\s\S]*?---\n?/, "")
|
|
323
|
+
.replace(/<!--\{[\s\S]*?\}-->/g, "")
|
|
324
|
+
.replace(/<!--[\s\S]*?-->/g, "");
|
|
325
|
+
|
|
326
|
+
if (manifestBaseUrl) {
|
|
327
|
+
const base = manifestBaseUrl.replace(/\/+$/, "");
|
|
328
|
+
cleaned = cleaned.replace(
|
|
329
|
+
/!\[(.*?)\]\((?!https?:\/\/|\/)(.*?)\)/g,
|
|
330
|
+
(_, alt, imgPath) => `})`,
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
return normalizeHeadings(cleaned, title);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Normalizes headings: drops redundant leading H1 matching title, demotes remaining headings.
|
|
339
|
+
* @param {string} content
|
|
340
|
+
* @param {string} title
|
|
341
|
+
* @returns {string}
|
|
342
|
+
*/
|
|
343
|
+
export function normalizeHeadings(content, title) {
|
|
344
|
+
const normalizedTitle = normalizeWhitespace(title).toLowerCase();
|
|
345
|
+
const lines = content.split("\n");
|
|
346
|
+
const processedLines = [];
|
|
347
|
+
let droppedLeadingH1 = false;
|
|
348
|
+
|
|
349
|
+
for (const line of lines) {
|
|
350
|
+
const h1Match = line.match(/^#\s+(.*)$/);
|
|
351
|
+
if (
|
|
352
|
+
!droppedLeadingH1 &&
|
|
353
|
+
h1Match &&
|
|
354
|
+
normalizeWhitespace(h1Match[1]).toLowerCase() === normalizedTitle
|
|
355
|
+
) {
|
|
356
|
+
droppedLeadingH1 = true;
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
processedLines.push(/^#{1,5}\s+/.test(line) ? `#${line}` : line);
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return processedLines.join("\n").trim();
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Maps items with a concurrency limit.
|
|
367
|
+
* @template T, R
|
|
368
|
+
* @param {T[]} items
|
|
369
|
+
* @param {number} limit
|
|
370
|
+
* @param {(item: T) => Promise<R>} iterator
|
|
371
|
+
* @returns {Promise<R[]>}
|
|
372
|
+
*/
|
|
373
|
+
export async function mapConcurrent(items, limit, iterator) {
|
|
374
|
+
const results = new Array(items.length);
|
|
375
|
+
let currentIndex = 0;
|
|
376
|
+
|
|
377
|
+
const workers = Array.from(
|
|
378
|
+
{ length: Math.min(limit, items.length) },
|
|
379
|
+
async () => {
|
|
380
|
+
while (currentIndex < items.length) {
|
|
381
|
+
const index = currentIndex++;
|
|
382
|
+
results[index] = await iterator(items[index]);
|
|
383
|
+
}
|
|
384
|
+
},
|
|
385
|
+
);
|
|
386
|
+
|
|
387
|
+
await Promise.all(workers);
|
|
388
|
+
return results;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Scans markdown and script files in srcDir to discover candidate story manifest URLs.
|
|
393
|
+
* @param {string} srcDir
|
|
394
|
+
* @returns {Promise<Array<{ url: string, filePath: string }>>}
|
|
395
|
+
*/
|
|
396
|
+
export async function detectManifestCandidates(srcDir) {
|
|
397
|
+
const candidates = new Map();
|
|
398
|
+
const urlRegex = /https?:\/\/[^\s'"\)\`]+(?:stories|narratives)\.json/g;
|
|
399
|
+
|
|
400
|
+
async function scanDir(dir) {
|
|
401
|
+
try {
|
|
402
|
+
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
403
|
+
for (const entry of entries) {
|
|
404
|
+
const fullPath = path.join(dir, entry.name);
|
|
405
|
+
if (
|
|
406
|
+
entry.isDirectory() &&
|
|
407
|
+
entry.name !== "node_modules" &&
|
|
408
|
+
entry.name !== ".vitepress"
|
|
409
|
+
) {
|
|
410
|
+
await scanDir(fullPath);
|
|
411
|
+
} else if (entry.isFile() && /\.(md|js|mjs|ts)$/.test(entry.name)) {
|
|
412
|
+
const content = await fs.readFile(fullPath, "utf8");
|
|
413
|
+
const matches = content.match(urlRegex) || [];
|
|
414
|
+
matches.forEach((url) => {
|
|
415
|
+
if (!candidates.has(url)) {
|
|
416
|
+
candidates.set(url, fullPath);
|
|
417
|
+
}
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
} catch {
|
|
422
|
+
// Ignore read errors gracefully
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
await scanDir(srcDir);
|
|
427
|
+
return Array.from(candidates.entries()).map(([url, filePath]) => ({
|
|
428
|
+
url,
|
|
429
|
+
filePath,
|
|
430
|
+
}));
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Fetches and parses a manifest JSON array.
|
|
435
|
+
* @param {string} url
|
|
436
|
+
* @returns {Promise<any[] | null>}
|
|
437
|
+
*/
|
|
438
|
+
export async function fetchManifest(url) {
|
|
439
|
+
try {
|
|
440
|
+
const res = await fetch(url);
|
|
441
|
+
if (!res.ok) return null;
|
|
442
|
+
const data = await res.json();
|
|
443
|
+
return Array.isArray(data) ? data : null;
|
|
444
|
+
} catch {
|
|
445
|
+
return null;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
/**
|
|
450
|
+
* Resolves manifest URL from options, brand config, or source files, and fetches the manifest.
|
|
451
|
+
* @param {object} params
|
|
452
|
+
* @param {string} [params.explicitUrl]
|
|
453
|
+
* @param {string | object} [params.brandStories]
|
|
454
|
+
* @param {string} params.srcDir
|
|
455
|
+
* @returns {Promise<{ manifestUrl: string, manifest: any[], sourceMarkdownPath: string | null } | null>}
|
|
456
|
+
*/
|
|
457
|
+
export async function resolveAndFetchManifest({
|
|
458
|
+
explicitUrl,
|
|
459
|
+
brandStories,
|
|
460
|
+
srcDir,
|
|
461
|
+
}) {
|
|
462
|
+
let manifestUrl = explicitUrl || null;
|
|
463
|
+
|
|
464
|
+
if (!manifestUrl && brandStories) {
|
|
465
|
+
manifestUrl =
|
|
466
|
+
typeof brandStories === "string" ? brandStories : brandStories.manifest;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
if (manifestUrl) {
|
|
470
|
+
const manifest = await fetchManifest(manifestUrl);
|
|
471
|
+
return manifest
|
|
472
|
+
? { manifestUrl, manifest, sourceMarkdownPath: null }
|
|
473
|
+
: null;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
const candidates = await detectManifestCandidates(srcDir);
|
|
477
|
+
if (candidates.length === 1) {
|
|
478
|
+
const { url, filePath } = candidates[0];
|
|
479
|
+
const manifest = await fetchManifest(url);
|
|
480
|
+
if (manifest) {
|
|
481
|
+
logger.info(`Detected stories manifest: ${pc.cyan(url)}`);
|
|
482
|
+
return { manifestUrl: url, manifest, sourceMarkdownPath: filePath };
|
|
483
|
+
}
|
|
484
|
+
} else if (candidates.length > 1) {
|
|
485
|
+
logger.warn(
|
|
486
|
+
`Multiple story manifest candidates detected; skipping: ${candidates.map((c) => c.url).join(", ")}`,
|
|
487
|
+
);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/**
|
|
494
|
+
* Fetches and processes story records concurrently.
|
|
495
|
+
* @param {object} params
|
|
496
|
+
* @returns {Promise<any[]>}
|
|
497
|
+
*/
|
|
498
|
+
export async function fetchStories({
|
|
499
|
+
manifest,
|
|
500
|
+
manifestBaseUrl,
|
|
501
|
+
base = "/",
|
|
502
|
+
routing = { isDynamic: false, routeDir: "story", routePath: "/story" },
|
|
503
|
+
domain = null,
|
|
504
|
+
concurrency = 8,
|
|
505
|
+
}) {
|
|
506
|
+
const stories = await mapConcurrent(manifest, concurrency, (entry, index) =>
|
|
507
|
+
fetchStory({
|
|
508
|
+
entry,
|
|
509
|
+
index,
|
|
510
|
+
manifestBaseUrl,
|
|
511
|
+
base,
|
|
512
|
+
routing,
|
|
513
|
+
domain,
|
|
514
|
+
}),
|
|
515
|
+
);
|
|
516
|
+
|
|
517
|
+
return stories.filter(Boolean);
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Fetches and processes an individual story.
|
|
522
|
+
* @param {object} params
|
|
523
|
+
* @returns {Promise<object | null>}
|
|
524
|
+
*/
|
|
525
|
+
export async function fetchStory({
|
|
526
|
+
entry,
|
|
527
|
+
index,
|
|
528
|
+
manifestBaseUrl,
|
|
529
|
+
base = "/",
|
|
530
|
+
routing = { isDynamic: false, routeDir: "story", routePath: "/story" },
|
|
531
|
+
domain = null,
|
|
532
|
+
}) {
|
|
533
|
+
if (!entry || typeof entry !== "object" || !entry.file || !entry.title) {
|
|
534
|
+
logger.warn(`Skipping invalid story at index ${index}`);
|
|
535
|
+
return null;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
const title = normalizeWhitespace(entry.title);
|
|
539
|
+
const subtitle = normalizeWhitespace(entry.subtitle);
|
|
540
|
+
const storyId = deriveStoryId(entry.file);
|
|
541
|
+
const markdownUrl = /^https?:\/\//.test(entry.file)
|
|
542
|
+
? entry.file
|
|
543
|
+
: `${manifestBaseUrl}/${entry.file.replace(/^\.\//, "")}`;
|
|
544
|
+
|
|
545
|
+
const { isDynamic, routeDir, routePath } = routing;
|
|
546
|
+
|
|
547
|
+
const storyUrl = buildStoryUrl({
|
|
548
|
+
domain,
|
|
549
|
+
base,
|
|
550
|
+
routeDir,
|
|
551
|
+
storyPath: routePath,
|
|
552
|
+
storyId,
|
|
553
|
+
isDynamic,
|
|
554
|
+
extension: "",
|
|
555
|
+
});
|
|
556
|
+
|
|
557
|
+
const cleanedBody = await fetchStoryMarkdown(
|
|
558
|
+
markdownUrl,
|
|
559
|
+
title,
|
|
560
|
+
manifestBaseUrl,
|
|
561
|
+
);
|
|
562
|
+
|
|
563
|
+
const docUrl =
|
|
564
|
+
isDynamic && cleanedBody
|
|
565
|
+
? buildStoryUrl({
|
|
566
|
+
domain,
|
|
567
|
+
base,
|
|
568
|
+
routeDir,
|
|
569
|
+
storyPath: routePath,
|
|
570
|
+
storyId,
|
|
571
|
+
isDynamic: true,
|
|
572
|
+
extension: ".md",
|
|
573
|
+
})
|
|
574
|
+
: markdownUrl;
|
|
575
|
+
|
|
576
|
+
return {
|
|
577
|
+
title,
|
|
578
|
+
subtitle,
|
|
579
|
+
storyId,
|
|
580
|
+
markdownUrl,
|
|
581
|
+
docUrl,
|
|
582
|
+
storyUrl,
|
|
583
|
+
cleanedBody,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/**
|
|
588
|
+
* Fetches and sanitizes markdown for a story.
|
|
589
|
+
* @param {string} markdownUrl
|
|
590
|
+
* @param {string} title
|
|
591
|
+
* @param {string} manifestBaseUrl
|
|
592
|
+
* @returns {Promise<string | null>}
|
|
593
|
+
*/
|
|
594
|
+
export async function fetchStoryMarkdown(markdownUrl, title, manifestBaseUrl) {
|
|
595
|
+
try {
|
|
596
|
+
const res = await fetch(markdownUrl);
|
|
597
|
+
if (!res.ok) {
|
|
598
|
+
logger.warn(`Failed to fetch story body: ${markdownUrl} (${res.status})`);
|
|
599
|
+
return null;
|
|
600
|
+
}
|
|
601
|
+
const text = await res.text();
|
|
602
|
+
return cleanStory(text, title, manifestBaseUrl);
|
|
603
|
+
} catch {
|
|
604
|
+
logger.warn(`Network error fetching story: ${markdownUrl}`);
|
|
605
|
+
return null;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Appends stories index section to llms.txt if it exists.
|
|
611
|
+
* @param {object} params
|
|
612
|
+
*/
|
|
613
|
+
export async function appendToLlmsTxt({ outDir, heading, stories }) {
|
|
614
|
+
const llmsTxtPath = path.resolve(outDir, "llms.txt");
|
|
615
|
+
let existing;
|
|
616
|
+
try {
|
|
617
|
+
existing = await fs.readFile(llmsTxtPath, "utf8");
|
|
618
|
+
} catch (err) {
|
|
619
|
+
if (err?.code !== "ENOENT") {
|
|
620
|
+
logger.warn(`Failed to read llms.txt: ${err?.message || err}`);
|
|
621
|
+
}
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
const headingRegex = new RegExp(`^##\\s+${escapeRegex(heading)}\\s*$`, "m");
|
|
626
|
+
if (headingRegex.test(existing)) {
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
const indexLines = stories.map((e) => {
|
|
631
|
+
const targetUrl = !e.cleanedBody
|
|
632
|
+
? e.markdownUrl
|
|
633
|
+
: e.docUrl || e.markdownUrl;
|
|
634
|
+
const desc = e.subtitle
|
|
635
|
+
? `${e.subtitle}. Story page: ${e.storyUrl}`
|
|
636
|
+
: `Story page: ${e.storyUrl}`;
|
|
637
|
+
return `- [${e.title}](${targetUrl}): ${desc}`;
|
|
638
|
+
});
|
|
639
|
+
const appendContent = `\n\n## ${heading}\n\n${indexLines.join("\n")}\n`;
|
|
640
|
+
try {
|
|
641
|
+
await fs.appendFile(llmsTxtPath, appendContent, "utf8");
|
|
642
|
+
} catch (err) {
|
|
643
|
+
logger.warn(`Failed to append to llms.txt: ${err?.message || err}`);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Appends full stories text to llms-full.txt if it exists.
|
|
649
|
+
* @param {object} params
|
|
650
|
+
*/
|
|
651
|
+
export async function appendToLlmsFullTxt({ outDir, heading, stories }) {
|
|
652
|
+
const llmsFullTxtPath = path.resolve(outDir, "llms-full.txt");
|
|
653
|
+
let existing;
|
|
654
|
+
try {
|
|
655
|
+
existing = await fs.readFile(llmsFullTxtPath, "utf8");
|
|
656
|
+
} catch (err) {
|
|
657
|
+
if (err?.code !== "ENOENT") {
|
|
658
|
+
logger.warn(`Failed to read llms-full.txt: ${err?.message || err}`);
|
|
659
|
+
}
|
|
660
|
+
return;
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
const headingRegex = new RegExp(`^#\\s+${escapeRegex(heading)}\\s*$`, "m");
|
|
664
|
+
if (headingRegex.test(existing)) {
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
const fullSections = stories
|
|
669
|
+
.filter((e) => e.cleanedBody)
|
|
670
|
+
.map(
|
|
671
|
+
(e) =>
|
|
672
|
+
`## ${e.title}\n\nSource: ${e.docUrl || e.markdownUrl}\nStory: ${e.storyUrl}\n\n${e.cleanedBody}`,
|
|
673
|
+
);
|
|
674
|
+
|
|
675
|
+
if (fullSections.length > 0) {
|
|
676
|
+
const appendContent = `\n\n# ${heading}\n\n${fullSections.join("\n\n---\n\n")}\n`;
|
|
677
|
+
try {
|
|
678
|
+
await fs.appendFile(llmsFullTxtPath, appendContent, "utf8");
|
|
679
|
+
} catch (err) {
|
|
680
|
+
logger.warn(`Failed to append to llms-full.txt: ${err?.message || err}`);
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/**
|
|
686
|
+
* Appends stories index section to the generated story markdown page in outDir.
|
|
687
|
+
* @param {object} params
|
|
688
|
+
*/
|
|
689
|
+
export async function appendToStoryRoutePage({
|
|
690
|
+
outDir,
|
|
691
|
+
heading,
|
|
692
|
+
stories,
|
|
693
|
+
sourceMarkdownPath = null,
|
|
694
|
+
srcDir = null,
|
|
695
|
+
routing = { isDynamic: false, routeDir: "story", routePath: "/story" },
|
|
696
|
+
}) {
|
|
697
|
+
let targetStoryPath = null;
|
|
698
|
+
const { isDynamic, routeDir, routePath } = routing;
|
|
699
|
+
|
|
700
|
+
if (sourceMarkdownPath && srcDir) {
|
|
701
|
+
const relativePath = path.relative(srcDir, sourceMarkdownPath);
|
|
702
|
+
if (!/\[.+?\]/.test(relativePath) && relativePath.endsWith(".md")) {
|
|
703
|
+
const candidatePath = path.resolve(outDir, relativePath);
|
|
704
|
+
try {
|
|
705
|
+
await fs.access(candidatePath);
|
|
706
|
+
targetStoryPath = candidatePath;
|
|
707
|
+
} catch {
|
|
708
|
+
// Fall through to fallback paths
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
if (!targetStoryPath) {
|
|
714
|
+
const cleanRoute = (isDynamic ? routeDir : routePath)
|
|
715
|
+
.replace(/\\/g, "/")
|
|
716
|
+
.replace(/^\/+|\/+$/g, "")
|
|
717
|
+
.replace(/\.md$/, "");
|
|
718
|
+
const candidatePaths = cleanRoute
|
|
719
|
+
? [
|
|
720
|
+
path.resolve(outDir, `${cleanRoute}.md`),
|
|
721
|
+
path.resolve(outDir, cleanRoute, "index.md"),
|
|
722
|
+
]
|
|
723
|
+
: isDynamic
|
|
724
|
+
? []
|
|
725
|
+
: [path.resolve(outDir, "index.md")];
|
|
726
|
+
|
|
727
|
+
for (const candidate of candidatePaths) {
|
|
728
|
+
try {
|
|
729
|
+
await fs.access(candidate);
|
|
730
|
+
targetStoryPath = candidate;
|
|
731
|
+
break;
|
|
732
|
+
} catch {
|
|
733
|
+
// Continue
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
if (!targetStoryPath) return;
|
|
739
|
+
|
|
740
|
+
const indexLines = stories.map((e) => {
|
|
741
|
+
const targetUrl = !e.cleanedBody
|
|
742
|
+
? e.markdownUrl
|
|
743
|
+
: e.docUrl || e.markdownUrl;
|
|
744
|
+
const desc = e.subtitle
|
|
745
|
+
? `${e.subtitle}. [View story](${e.storyUrl})`
|
|
746
|
+
: `[View story](${e.storyUrl})`;
|
|
747
|
+
return `- [${e.title}](${targetUrl}) — ${desc}`;
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
let existing;
|
|
751
|
+
try {
|
|
752
|
+
existing = await fs.readFile(targetStoryPath, "utf8");
|
|
753
|
+
} catch (err) {
|
|
754
|
+
if (err?.code !== "ENOENT") {
|
|
755
|
+
logger.warn(`Failed to read story route page: ${err?.message || err}`);
|
|
756
|
+
}
|
|
757
|
+
return;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
const headingRegex = new RegExp(`^##\\s+${escapeRegex(heading)}\\s*$`, "m");
|
|
761
|
+
if (headingRegex.test(existing)) {
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
const appendContent = `\n\n## ${heading}\n\n${indexLines.join("\n")}\n`;
|
|
766
|
+
try {
|
|
767
|
+
await fs.appendFile(targetStoryPath, appendContent, "utf8");
|
|
768
|
+
} catch (err) {
|
|
769
|
+
logger.warn(`Failed to append to story route page: ${err?.message || err}`);
|
|
770
|
+
}
|
|
771
|
+
}
|