@nuasite/llm-enhancements 0.64.3 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{types/build-processor.d.ts → build-processor.d.ts} +1 -1
- package/dist/build-processor.d.ts.map +1 -0
- package/dist/build-processor.js +135 -0
- package/dist/{types/cms-marker.d.ts → cms-marker.d.ts} +1 -1
- package/dist/cms-marker.d.ts.map +1 -0
- package/dist/cms-marker.js +59 -0
- package/dist/cms.d.ts.map +1 -0
- package/dist/cms.js +21 -0
- package/dist/{types/dev-middleware.d.ts → dev-middleware.d.ts} +1 -1
- package/dist/dev-middleware.d.ts.map +1 -0
- package/dist/dev-middleware.js +216 -0
- package/dist/html-to-markdown.d.ts.map +1 -0
- package/dist/html-to-markdown.js +298 -0
- package/dist/{types/index.d.ts → index.d.ts} +2 -2
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/{types/llm-endpoint.d.ts → llm-endpoint.d.ts} +1 -1
- package/dist/llm-endpoint.d.ts.map +1 -0
- package/dist/llm-endpoint.js +52 -0
- package/dist/{types/llms-txt-endpoint.d.ts → llms-txt-endpoint.d.ts} +1 -1
- package/dist/llms-txt-endpoint.d.ts.map +1 -0
- package/dist/llms-txt-endpoint.js +92 -0
- package/dist/{types/markdown-generator.d.ts → markdown-generator.d.ts} +1 -1
- package/dist/markdown-generator.d.ts.map +1 -0
- package/dist/markdown-generator.js +106 -0
- package/dist/mdx.d.ts.map +1 -0
- package/dist/mdx.js +90 -0
- package/dist/{types/page-markdown.d.ts → page-markdown.d.ts} +4 -4
- package/dist/page-markdown.d.ts.map +1 -0
- package/dist/page-markdown.js +34 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +79 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/dist/types/tests/tsconfig.tsbuildinfo +1 -1
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +11 -0
- package/package.json +9 -9
- package/src/build-processor.ts +10 -10
- package/src/cms-marker.ts +3 -3
- package/src/dev-middleware.ts +10 -10
- package/src/index.ts +4 -4
- package/src/llm-endpoint.ts +2 -2
- package/src/llms-txt-endpoint.ts +2 -2
- package/src/markdown-generator.ts +1 -1
- package/src/page-markdown.ts +5 -5
- package/src/tsconfig.json +5 -1
- package/dist/types/build-processor.d.ts.map +0 -1
- package/dist/types/cms-marker.d.ts.map +0 -1
- package/dist/types/cms.d.ts.map +0 -1
- package/dist/types/dev-middleware.d.ts.map +0 -1
- package/dist/types/html-to-markdown.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/llm-endpoint.d.ts.map +0 -1
- package/dist/types/llms-txt-endpoint.d.ts.map +0 -1
- package/dist/types/markdown-generator.d.ts.map +0 -1
- package/dist/types/mdx.d.ts.map +0 -1
- package/dist/types/page-markdown.d.ts.map +0 -1
- package/dist/types/paths.d.ts.map +0 -1
- package/dist/types/tsconfig.tsbuildinfo +0 -1
- package/dist/types/types.d.ts.map +0 -1
- /package/dist/{types/cms.d.ts → cms.d.ts} +0 -0
- /package/dist/{types/html-to-markdown.d.ts → html-to-markdown.d.ts} +0 -0
- /package/dist/{types/mdx.d.ts → mdx.d.ts} +0 -0
- /package/dist/{types/paths.d.ts → paths.d.ts} +0 -0
- /package/dist/{types/types.d.ts → types.d.ts} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build-processor.d.ts","sourceRoot":"","sources":["../src/build-processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,OAAO,CAAA;AAahE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjD,UAAU,QAAQ;IACjB,QAAQ,EAAE,MAAM,CAAA;CAChB;AAWD;;;GAGG;AACH,wBAAgB,kBAAkB,CACjC,GAAG,EAAE,GAAG,EACR,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,sBAAsB,EAC9B,MAAM,EAAE,WAAW,GACjB,OAAO,CAAC,IAAI,CAAC,CAGf"}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { findCollectionPages } from "./cms-marker.js";
|
|
5
|
+
import { cms } from "./cms.js";
|
|
6
|
+
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
7
|
+
import { generateLlmMarkdown } from "./llm-endpoint.js";
|
|
8
|
+
import { generateLlmsTxt } from "./llms-txt-endpoint.js";
|
|
9
|
+
import { generateMarkdown } from "./markdown-generator.js";
|
|
10
|
+
import { markdownSyntaxOf } from "./mdx.js";
|
|
11
|
+
import { resolvePageMarkdown } from "./page-markdown.js";
|
|
12
|
+
import { getHtmlPath, getLlmOutputPath, getLlmsTxtOutputPath, getMdOutputPath, injectMarkdownLink, normalizePath } from "./paths.js";
|
|
13
|
+
/**
|
|
14
|
+
* Get base URL from Astro config, removing trailing slash
|
|
15
|
+
*/
|
|
16
|
+
function getBaseUrl(config) {
|
|
17
|
+
const site = config.site;
|
|
18
|
+
if (!site)
|
|
19
|
+
return '';
|
|
20
|
+
return site.endsWith('/') ? site.slice(0, -1) : site;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Process build output and generate .md files for all pages. Entries are read from Astro's project
|
|
24
|
+
* root, which need not be the working directory (`astro build --root site`).
|
|
25
|
+
*/
|
|
26
|
+
export function processBuildOutput(dir, pages, options, logger, config) {
|
|
27
|
+
const run = () => processPages(dir, pages, options, logger, config);
|
|
28
|
+
return cms ? cms.runAtProjectRoot(fileURLToPath(config.root), run) : run();
|
|
29
|
+
}
|
|
30
|
+
async function processPages(dir, pages, options, logger, config) {
|
|
31
|
+
const baseUrl = getBaseUrl(config);
|
|
32
|
+
const distDir = fileURLToPath(dir);
|
|
33
|
+
let collectionCount = 0;
|
|
34
|
+
let staticCount = 0;
|
|
35
|
+
const pageEntries = [];
|
|
36
|
+
let siteMetadata = {};
|
|
37
|
+
// Found once for the whole build: resolving a page against a fresh scan made the build quadratic.
|
|
38
|
+
const site = {
|
|
39
|
+
pages: await findCollectionPages(options.contentDir, message => logger.warn(message)),
|
|
40
|
+
syntax: markdownSyntaxOf(config),
|
|
41
|
+
};
|
|
42
|
+
for (const page of pages) {
|
|
43
|
+
const pagePath = normalizePath(page.pathname === '' ? '/' : `/${page.pathname}`);
|
|
44
|
+
try {
|
|
45
|
+
const mdPath = getMdOutputPath(distDir, pagePath);
|
|
46
|
+
const htmlPath = getHtmlPath(distDir, pagePath);
|
|
47
|
+
// Read once, for the markdown and for the link injected into it.
|
|
48
|
+
let html;
|
|
49
|
+
const readHtml = () => (html ??= fs.readFile(htmlPath, 'utf-8').catch(() => undefined));
|
|
50
|
+
const resolved = await resolvePageMarkdown(pagePath, options, site, async () => {
|
|
51
|
+
const page = await readHtml();
|
|
52
|
+
return page === undefined ? undefined : htmlToMarkdown(page);
|
|
53
|
+
});
|
|
54
|
+
if (!resolved)
|
|
55
|
+
continue;
|
|
56
|
+
const markdown = generateMarkdown(resolved.output, {
|
|
57
|
+
url: pagePath,
|
|
58
|
+
type: resolved.type,
|
|
59
|
+
sourcePath: resolved.sourcePath,
|
|
60
|
+
baseUrl,
|
|
61
|
+
}, options.includeFrontmatter);
|
|
62
|
+
await writeMarkdownFile(mdPath, markdown);
|
|
63
|
+
pageEntries.push({ pathname: pagePath, title: resolved.title, type: resolved.type });
|
|
64
|
+
if (resolved.type === 'collection')
|
|
65
|
+
collectionCount++;
|
|
66
|
+
else
|
|
67
|
+
staticCount++;
|
|
68
|
+
// The site's title and description, from the rendered homepage, also when an entry tells it.
|
|
69
|
+
if (pagePath === '/') {
|
|
70
|
+
const page = resolved.metadata ? undefined : await readHtml();
|
|
71
|
+
const metadata = resolved.metadata ?? (page === undefined ? undefined : htmlToMarkdown(page).metadata);
|
|
72
|
+
if (metadata)
|
|
73
|
+
siteMetadata = { title: metadata.title, description: metadata.description };
|
|
74
|
+
}
|
|
75
|
+
// The markdown is written whether or not its link can be added to the page.
|
|
76
|
+
try {
|
|
77
|
+
await injectLinkIntoHtml(htmlPath, pagePath, await readHtml());
|
|
78
|
+
}
|
|
79
|
+
catch (error) {
|
|
80
|
+
logger.warn(`Failed to link ${pagePath} to its markdown: ${error}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
logger.warn(`Failed to process ${pagePath}: ${error}`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const total = collectionCount + staticCount;
|
|
88
|
+
if (total > 0) {
|
|
89
|
+
logger.info(`Generated ${total} .md files (${collectionCount} collection, ${staticCount} static)`);
|
|
90
|
+
}
|
|
91
|
+
// Generate llm.md if enabled
|
|
92
|
+
if (options.llmEndpoint !== false) {
|
|
93
|
+
if (!baseUrl) {
|
|
94
|
+
logger.warn('Skipping /.well-known/llm.md generation: no `site` configured in astro.config');
|
|
95
|
+
}
|
|
96
|
+
else {
|
|
97
|
+
try {
|
|
98
|
+
const llmContent = generateLlmMarkdown(pageEntries, { ...siteMetadata, baseUrl }, options.llmEndpoint);
|
|
99
|
+
const llmPath = getLlmOutputPath(distDir);
|
|
100
|
+
await writeMarkdownFile(llmPath, llmContent);
|
|
101
|
+
logger.info('Generated /.well-known/llm.md');
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
logger.warn(`Failed to generate llm.md: ${error}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
// Generate llms.txt if enabled
|
|
109
|
+
if (options.llmsTxt !== false) {
|
|
110
|
+
if (!baseUrl) {
|
|
111
|
+
logger.warn('Skipping /llms.txt generation: no `site` configured in astro.config');
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
try {
|
|
115
|
+
const llmsTxtContent = generateLlmsTxt(pageEntries, { ...siteMetadata, baseUrl }, options.llmsTxt);
|
|
116
|
+
const llmsTxtPath = getLlmsTxtOutputPath(distDir);
|
|
117
|
+
await writeMarkdownFile(llmsTxtPath, llmsTxtContent);
|
|
118
|
+
logger.info('Generated /llms.txt');
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
logger.warn(`Failed to generate llms.txt: ${error}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
async function injectLinkIntoHtml(htmlPath, pagePath, html) {
|
|
127
|
+
// Some pages have no HTML file
|
|
128
|
+
if (html === undefined)
|
|
129
|
+
return;
|
|
130
|
+
await fs.writeFile(htmlPath, injectMarkdownLink(html, pagePath), 'utf-8');
|
|
131
|
+
}
|
|
132
|
+
async function writeMarkdownFile(filePath, content) {
|
|
133
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
134
|
+
await fs.writeFile(filePath, content, 'utf-8');
|
|
135
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CollectionInfo, MarkdownContent } from '@nuasite/cms';
|
|
2
|
-
import { type MarkdownSyntax } from './mdx';
|
|
2
|
+
import { type MarkdownSyntax } from './mdx.ts';
|
|
3
3
|
/** The pages that render a collection entry, by normalized pathname. */
|
|
4
4
|
export type CollectionPages = Map<string, CollectionInfo>;
|
|
5
5
|
/**
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cms-marker.d.ts","sourceRoot":"","sources":["../src/cms-marker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAGnE,OAAO,EAAE,KAAK,cAAc,EAAqB,MAAM,UAAU,CAAA;AAGjE,wEAAwE;AACxE,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;AAEzD;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,MAAM,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAA;AAQzF,wBAAsB,oBAAoB,CACzC,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,eAAe,EACtB,MAAM,EAAE,cAAc,GACpB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAmBrC;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC,CAoBvH"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { cms } from "./cms.js";
|
|
3
|
+
import { mdxBodyAsMarkdown } from "./mdx.js";
|
|
4
|
+
import { normalizePath } from "./paths.js";
|
|
5
|
+
/**
|
|
6
|
+
* Entries whose body is markdown. Any other entry (`.json`, `.yaml`, Markdoc) still owns its page, but
|
|
7
|
+
* is told by the rendered HTML, like an `.mdx` entry that renders components.
|
|
8
|
+
*/
|
|
9
|
+
const MARKDOWN_ENTRY = /\.(md|mdx|markdown)$/i;
|
|
10
|
+
export async function getCollectionContent(pagePath, pages, syntax) {
|
|
11
|
+
const collectionInfo = pages.get(normalizePath(pagePath));
|
|
12
|
+
if (!cms || !collectionInfo)
|
|
13
|
+
return undefined;
|
|
14
|
+
if (!MARKDOWN_ENTRY.test(collectionInfo.file)) {
|
|
15
|
+
return {
|
|
16
|
+
frontmatter: {},
|
|
17
|
+
body: undefined,
|
|
18
|
+
bodyStartLine: 0,
|
|
19
|
+
file: collectionInfo.file,
|
|
20
|
+
collectionName: collectionInfo.name,
|
|
21
|
+
collectionSlug: collectionInfo.slug,
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
const content = await cms.parseMarkdownContent(collectionInfo);
|
|
25
|
+
if (!content)
|
|
26
|
+
return undefined;
|
|
27
|
+
const body = content.body.replace(/\r\n?/g, '\n');
|
|
28
|
+
return { ...content, body: content.file.toLowerCase().endsWith('.mdx') ? mdxBodyAsMarkdown(body, syntax) : body };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Find the pages that render a collection entry, once, for a caller about to resolve many pages: the URL
|
|
32
|
+
* the CMS editor shows for each entry (see `collectionPagesByPathname` in `@nuasite/cms`). Runs in the
|
|
33
|
+
* current project context. Empty without `@nuasite/cms`, when `contentDir` lies outside the project root
|
|
34
|
+
* (the CMS reads nothing there), or when the scan fails; `warn` reports the last two. Every page is then
|
|
35
|
+
* told by its rendered HTML.
|
|
36
|
+
*/
|
|
37
|
+
export async function findCollectionPages(contentDir, warn) {
|
|
38
|
+
if (!cms)
|
|
39
|
+
return new Map();
|
|
40
|
+
const root = cms.getProjectRoot();
|
|
41
|
+
const relative = path.relative(root, path.resolve(root, contentDir));
|
|
42
|
+
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
43
|
+
warn(`contentDir ${contentDir} is outside the project root ${root}, every page is converted from its HTML`);
|
|
44
|
+
return new Map();
|
|
45
|
+
}
|
|
46
|
+
let found;
|
|
47
|
+
try {
|
|
48
|
+
// Relative, as the CMS spells the source paths it reads entries from.
|
|
49
|
+
found = await cms.collectionPagesByPathname(await cms.scanProjectCollections(relative.split(path.sep).join('/') || '.'));
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
warn(`Could not scan the content collections in ${contentDir}, every page is converted from its HTML: ${error}`);
|
|
53
|
+
return new Map();
|
|
54
|
+
}
|
|
55
|
+
const pages = new Map();
|
|
56
|
+
for (const [pathname, info] of found)
|
|
57
|
+
pages.set(normalizePath(pathname), info);
|
|
58
|
+
return pages;
|
|
59
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cms.d.ts","sourceRoot":"","sources":["../src/cms.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,GAAG,MAAM,cAAc,CAAA;AAExC;;;;;;;GAOG;AACH,eAAO,MAAM,GAAG,EAAE,OAAO,GAAG,GAAG,SAG7B,CAAA"}
|
package/dist/cms.js
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@nuasite/cms`, an optional peer: without it every page is converted from its rendered HTML.
|
|
3
|
+
*
|
|
4
|
+
* Imported here, at the top level, and nowhere later. Astro evaluates an integration through a Vite
|
|
5
|
+
* module runner that it closes once the config is loaded, so an import made in a hook fails in any Astro
|
|
6
|
+
* build, Node or Bun ("Vite module runner has been closed"). A top-level await finishes while the config
|
|
7
|
+
* is still loading.
|
|
8
|
+
*/
|
|
9
|
+
export const cms = await import('@nuasite/cms').catch((error) => {
|
|
10
|
+
if (isMissing(error))
|
|
11
|
+
return undefined;
|
|
12
|
+
throw error;
|
|
13
|
+
});
|
|
14
|
+
/** The package is not installed, as opposed to installed and failing to load. */
|
|
15
|
+
function isMissing(error) {
|
|
16
|
+
const code = error?.code;
|
|
17
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
18
|
+
return (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || /Cannot find (module|package)/.test(message))
|
|
19
|
+
// Quoted, so a missing `@nuasite/cms-types` under an installed cms is not taken for cms itself.
|
|
20
|
+
&& /['"]@nuasite\/cms['"]/.test(message);
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dev-middleware.d.ts","sourceRoot":"","sources":["../src/dev-middleware.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AAGxC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,MAAM,CAAA;AAUzC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AA2EjD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,WAAW,QAsKvG"}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { createPublicStaticFileChecker } from '@nuasite/cms-core';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileURLToPath } from 'node:url';
|
|
4
|
+
import { findCollectionPages } from "./cms-marker.js";
|
|
5
|
+
import { cms } from "./cms.js";
|
|
6
|
+
import { htmlToMarkdown } from "./html-to-markdown.js";
|
|
7
|
+
import { generateLlmMarkdown } from "./llm-endpoint.js";
|
|
8
|
+
import { generateLlmsTxt } from "./llms-txt-endpoint.js";
|
|
9
|
+
import { generateMarkdown } from "./markdown-generator.js";
|
|
10
|
+
import { markdownSyntaxOf } from "./mdx.js";
|
|
11
|
+
import { resolvePageMarkdown } from "./page-markdown.js";
|
|
12
|
+
import { injectMarkdownLink, LLM_ENDPOINT_PATH, LLMS_TXT_PATH, mdUrlToPagePath, normalizePath } from "./paths.js";
|
|
13
|
+
const ASSET_PATTERN = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|json)$/;
|
|
14
|
+
/**
|
|
15
|
+
* Generate markdown for a given page path
|
|
16
|
+
*/
|
|
17
|
+
async function generateMarkdownForPath(pagePath, host, options, baseUrl, site) {
|
|
18
|
+
const resolved = await resolvePageMarkdown(pagePath, options, site, () => fetchPage(host, pagePath));
|
|
19
|
+
if (!resolved)
|
|
20
|
+
return null;
|
|
21
|
+
return generateMarkdown(resolved.output, {
|
|
22
|
+
url: pagePath,
|
|
23
|
+
type: resolved.type,
|
|
24
|
+
sourcePath: resolved.sourcePath,
|
|
25
|
+
baseUrl,
|
|
26
|
+
}, options.includeFrontmatter);
|
|
27
|
+
}
|
|
28
|
+
/** The dev server's rendering of a page, converted to markdown, or `undefined` when it serves no HTML there. */
|
|
29
|
+
async function fetchPage(host, pagePath) {
|
|
30
|
+
const response = await fetch(`http://${host}${pagePath}`, {
|
|
31
|
+
headers: { Accept: 'text/html' },
|
|
32
|
+
});
|
|
33
|
+
if (!response.ok)
|
|
34
|
+
return undefined;
|
|
35
|
+
const contentType = response.headers.get('content-type');
|
|
36
|
+
if (!contentType?.includes('text/html'))
|
|
37
|
+
return undefined;
|
|
38
|
+
return htmlToMarkdown(await response.text());
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Discover all pages and their metadata for the LLM endpoint
|
|
42
|
+
*/
|
|
43
|
+
async function discoverPages(host, options, site) {
|
|
44
|
+
const pages = [];
|
|
45
|
+
let siteMetadata = {};
|
|
46
|
+
// For now, we discover only the homepage: the dev server has no list of routes to offer. A homepage
|
|
47
|
+
// the dev server fails to render is still listed when its entry tells it.
|
|
48
|
+
const home = fetchPage(host, '/').catch(() => undefined);
|
|
49
|
+
const metadata = (await home)?.metadata;
|
|
50
|
+
if (metadata)
|
|
51
|
+
siteMetadata = { title: metadata.title, description: metadata.description };
|
|
52
|
+
try {
|
|
53
|
+
const resolved = await resolvePageMarkdown('/', options, site, () => home);
|
|
54
|
+
if (resolved)
|
|
55
|
+
pages.push({ pathname: '/', title: resolved.title, type: resolved.type });
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
// Ignore
|
|
59
|
+
}
|
|
60
|
+
return { pages, siteMetadata };
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Get base URL from Astro config, removing trailing slash
|
|
64
|
+
*/
|
|
65
|
+
function getBaseUrl(config) {
|
|
66
|
+
const site = config.site;
|
|
67
|
+
if (!site)
|
|
68
|
+
return '';
|
|
69
|
+
return site.endsWith('/') ? site.slice(0, -1) : site;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Create dev server middleware to handle markdown requests
|
|
73
|
+
*/
|
|
74
|
+
export function createDevMiddleware(server, options, config) {
|
|
75
|
+
const baseUrl = getBaseUrl(config);
|
|
76
|
+
// Entries are read from Astro's root, as in the build, whatever the working directory is. The project
|
|
77
|
+
// and the pages that render an entry are kept between requests, and replaced after a change to the
|
|
78
|
+
// content, the routes or the content config. The routes are the ones the CMS reads, in `src/pages`.
|
|
79
|
+
const root = fileURLToPath(config.root);
|
|
80
|
+
const watched = [path.resolve(root, options.contentDir) + path.sep, path.join(root, 'src', 'pages') + path.sep];
|
|
81
|
+
const syntax = markdownSyntaxOf(config);
|
|
82
|
+
let project = cms?.projectAtRoot(root);
|
|
83
|
+
let site;
|
|
84
|
+
server.watcher?.on('all', (_event, file) => {
|
|
85
|
+
const changed = path.resolve(file);
|
|
86
|
+
if (watched.some(dir => changed.startsWith(dir)) || /content\.config\.[cm]?[jt]s$/.test(changed)) {
|
|
87
|
+
project = cms?.projectAtRoot(root);
|
|
88
|
+
site = undefined;
|
|
89
|
+
}
|
|
90
|
+
});
|
|
91
|
+
/** Run `resolve` in the project, against the pages that render an entry. */
|
|
92
|
+
const withSite = (resolve) => {
|
|
93
|
+
const run = async () => {
|
|
94
|
+
site ??= findCollectionPages(options.contentDir, message => console.warn(`[llm-enhancements] ${message}`)).then(pages => ({ pages, syntax }));
|
|
95
|
+
return resolve(await site);
|
|
96
|
+
};
|
|
97
|
+
return project ? project.run(run) : run();
|
|
98
|
+
};
|
|
99
|
+
const isPublicStaticFile = createPublicStaticFileChecker(config.publicDir ? fileURLToPath(config.publicDir) : undefined);
|
|
100
|
+
// Serve /llms.txt endpoint (only if site is configured)
|
|
101
|
+
const llmsTxtOptions = options.llmsTxt;
|
|
102
|
+
if (llmsTxtOptions !== false && baseUrl) {
|
|
103
|
+
server.middlewares.use(async (req, res, next) => {
|
|
104
|
+
const url = req.url || '';
|
|
105
|
+
if (url !== LLMS_TXT_PATH) {
|
|
106
|
+
return next();
|
|
107
|
+
}
|
|
108
|
+
try {
|
|
109
|
+
const host = req.headers.host || 'localhost:4321';
|
|
110
|
+
const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site));
|
|
111
|
+
const content = generateLlmsTxt(pages, { ...siteMetadata, baseUrl }, llmsTxtOptions);
|
|
112
|
+
res.setHeader('Content-Type', 'text/plain; charset=utf-8');
|
|
113
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
114
|
+
res.end(content);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
console.error('[llm-enhancements] Error generating llms.txt:', error);
|
|
119
|
+
}
|
|
120
|
+
return next();
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
// Serve /.well-known/llm.md endpoint (only if site is configured)
|
|
124
|
+
const llmEndpointOptions = options.llmEndpoint;
|
|
125
|
+
if (llmEndpointOptions !== false && baseUrl) {
|
|
126
|
+
server.middlewares.use(async (req, res, next) => {
|
|
127
|
+
const url = req.url || '';
|
|
128
|
+
if (url !== LLM_ENDPOINT_PATH) {
|
|
129
|
+
return next();
|
|
130
|
+
}
|
|
131
|
+
try {
|
|
132
|
+
const host = req.headers.host || 'localhost:4321';
|
|
133
|
+
const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site));
|
|
134
|
+
const markdown = generateLlmMarkdown(pages, { ...siteMetadata, baseUrl }, llmEndpointOptions);
|
|
135
|
+
res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
|
|
136
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
137
|
+
res.end(markdown);
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
console.error('[llm-enhancements] Error generating llm.md:', error);
|
|
142
|
+
}
|
|
143
|
+
return next();
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
// Serve .md endpoints
|
|
147
|
+
server.middlewares.use(async (req, res, next) => {
|
|
148
|
+
const url = req.url || '';
|
|
149
|
+
if (!url.endsWith('.md')) {
|
|
150
|
+
return next();
|
|
151
|
+
}
|
|
152
|
+
// Decoded, as the pages that render an entry are keyed: `/aktuality/%C5%A1kola.md` is `/aktuality/škola`.
|
|
153
|
+
let pagePath;
|
|
154
|
+
try {
|
|
155
|
+
pagePath = mdUrlToPagePath(decodeURI(url));
|
|
156
|
+
}
|
|
157
|
+
catch {
|
|
158
|
+
return next();
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
const host = req.headers.host || 'localhost:4321';
|
|
162
|
+
const markdown = await withSite(site => generateMarkdownForPath(pagePath, host, options, baseUrl, site));
|
|
163
|
+
if (markdown) {
|
|
164
|
+
res.setHeader('Content-Type', 'text/markdown; charset=utf-8');
|
|
165
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
166
|
+
res.end(markdown);
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
console.error('[llm-enhancements] Error generating markdown:', error);
|
|
172
|
+
}
|
|
173
|
+
return next();
|
|
174
|
+
});
|
|
175
|
+
// Inject alternate link into HTML responses
|
|
176
|
+
server.middlewares.use((req, res, next) => {
|
|
177
|
+
const url = req.url || '';
|
|
178
|
+
if (url.endsWith('.md') || ASSET_PATTERN.test(url)) {
|
|
179
|
+
return next();
|
|
180
|
+
}
|
|
181
|
+
// Skip vendor HTML served from publicDir — it has no .md counterpart, and
|
|
182
|
+
// rewriting the body without updating content-length truncates the response.
|
|
183
|
+
if (isPublicStaticFile(url)) {
|
|
184
|
+
return next();
|
|
185
|
+
}
|
|
186
|
+
const originalWrite = res.write;
|
|
187
|
+
const originalEnd = res.end;
|
|
188
|
+
const chunks = [];
|
|
189
|
+
res.write = ((chunk) => {
|
|
190
|
+
if (chunk)
|
|
191
|
+
chunks.push(Buffer.from(chunk));
|
|
192
|
+
return true;
|
|
193
|
+
});
|
|
194
|
+
res.end = ((chunk, ...args) => {
|
|
195
|
+
if (chunk)
|
|
196
|
+
chunks.push(Buffer.from(chunk));
|
|
197
|
+
const contentType = res.getHeader('content-type');
|
|
198
|
+
const isHtml = typeof contentType === 'string' && contentType.includes('text/html');
|
|
199
|
+
res.write = originalWrite;
|
|
200
|
+
res.end = originalEnd;
|
|
201
|
+
if (isHtml && chunks.length > 0) {
|
|
202
|
+
const html = Buffer.concat(chunks).toString('utf8');
|
|
203
|
+
const pagePath = normalizePath(url);
|
|
204
|
+
const transformed = injectMarkdownLink(html, pagePath);
|
|
205
|
+
if (!res.headersSent) {
|
|
206
|
+
res.removeHeader('content-length');
|
|
207
|
+
}
|
|
208
|
+
return res.end(transformed, ...args);
|
|
209
|
+
}
|
|
210
|
+
return chunks.length > 0
|
|
211
|
+
? res.end(Buffer.concat(chunks), ...args)
|
|
212
|
+
: res.end(...args);
|
|
213
|
+
});
|
|
214
|
+
next();
|
|
215
|
+
});
|
|
216
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-to-markdown.d.ts","sourceRoot":"","sources":["../src/html-to-markdown.ts"],"names":[],"mappings":"AAkTA,MAAM,WAAW,oBAAoB;IACpC,yCAAyC;IACzC,QAAQ,EAAE;QACT,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,WAAW,CAAC,EAAE,MAAM,CAAA;KACpB,CAAA;IACD,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,CA+BjE"}
|