@nuasite/llm-enhancements 0.60.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -14
- package/dist/types/build-processor.d.ts +2 -1
- package/dist/types/build-processor.d.ts.map +1 -1
- package/dist/types/cms-marker.d.ts +21 -17
- package/dist/types/cms-marker.d.ts.map +1 -1
- package/dist/types/cms.d.ts +11 -0
- package/dist/types/cms.d.ts.map +1 -0
- package/dist/types/dev-middleware.d.ts.map +1 -1
- package/dist/types/mdx.d.ts +26 -0
- package/dist/types/mdx.d.ts.map +1 -0
- package/dist/types/page-markdown.d.ts +26 -0
- package/dist/types/page-markdown.d.ts.map +1 -0
- package/dist/types/tests/cases/build-processor.test.d.ts +2 -0
- package/dist/types/tests/cases/build-processor.test.d.ts.map +1 -0
- package/dist/types/tests/cases/dev-middleware.test.d.ts +2 -0
- package/dist/types/tests/cases/dev-middleware.test.d.ts.map +1 -0
- package/dist/types/tests/cases/mdx.test.d.ts +2 -0
- package/dist/types/tests/cases/mdx.test.d.ts.map +1 -0
- package/dist/types/tests/tsconfig.tsbuildinfo +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +8 -3
- package/src/build-processor.ts +54 -72
- package/src/cms-marker.ts +64 -41
- package/src/cms.ts +23 -0
- package/src/dev-middleware.ts +69 -68
- package/src/mdx.ts +103 -0
- package/src/page-markdown.ts +59 -0
- package/src/tsconfig.json +2 -1
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"directory": "packages/llm-enhancements"
|
|
15
15
|
},
|
|
16
16
|
"license": "Apache-2.0",
|
|
17
|
-
"version": "0.
|
|
17
|
+
"version": "0.61.0",
|
|
18
18
|
"module": "src/index.ts",
|
|
19
19
|
"types": "src/index.ts",
|
|
20
20
|
"type": "module",
|
|
@@ -26,7 +26,12 @@
|
|
|
26
26
|
}
|
|
27
27
|
},
|
|
28
28
|
"dependencies": {
|
|
29
|
-
"
|
|
29
|
+
"@nuasite/cms-core": "0.61.0",
|
|
30
|
+
"node-html-parser": "^7.1.0",
|
|
31
|
+
"remark-directive": "^3.0.1",
|
|
32
|
+
"remark-mdx": "^3.1.0",
|
|
33
|
+
"remark-parse": "^11.0.0",
|
|
34
|
+
"unified": "^11.0.5"
|
|
30
35
|
},
|
|
31
36
|
"devDependencies": {
|
|
32
37
|
"@types/bun": "1.3.11",
|
|
@@ -35,7 +40,7 @@
|
|
|
35
40
|
"peerDependencies": {
|
|
36
41
|
"typescript": "^6.0.2",
|
|
37
42
|
"vite": "^7.0.0",
|
|
38
|
-
"@nuasite/cms": "0.
|
|
43
|
+
"@nuasite/cms": "0.61.0"
|
|
39
44
|
},
|
|
40
45
|
"peerDependenciesMeta": {
|
|
41
46
|
"@nuasite/cms": {
|
package/src/build-processor.ts
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import type { AstroConfig, AstroIntegrationLogger } from 'astro'
|
|
2
2
|
import fs from 'node:fs/promises'
|
|
3
3
|
import path from 'node:path'
|
|
4
|
-
import {
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
import { cms } from './cms'
|
|
6
|
+
import { findCollectionPages } from './cms-marker'
|
|
5
7
|
import { htmlToMarkdown } from './html-to-markdown'
|
|
6
8
|
import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
|
|
7
9
|
import { generateLlmsTxt } from './llms-txt-endpoint'
|
|
8
|
-
import {
|
|
10
|
+
import { generateMarkdown } from './markdown-generator'
|
|
11
|
+
import { markdownSyntaxOf } from './mdx'
|
|
12
|
+
import { resolvePageMarkdown } from './page-markdown'
|
|
9
13
|
import { getHtmlPath, getLlmOutputPath, getLlmsTxtOutputPath, getMdOutputPath, injectMarkdownLink, normalizePath } from './paths'
|
|
10
14
|
import type { ResolvedOptions } from './types'
|
|
11
15
|
|
|
@@ -23,9 +27,21 @@ function getBaseUrl(config: AstroConfig): string {
|
|
|
23
27
|
}
|
|
24
28
|
|
|
25
29
|
/**
|
|
26
|
-
* Process build output and generate .md files for all pages
|
|
30
|
+
* Process build output and generate .md files for all pages. Entries are read from Astro's project
|
|
31
|
+
* root, which need not be the working directory (`astro build --root site`).
|
|
27
32
|
*/
|
|
28
|
-
export
|
|
33
|
+
export function processBuildOutput(
|
|
34
|
+
dir: URL,
|
|
35
|
+
pages: PageInfo[],
|
|
36
|
+
options: ResolvedOptions,
|
|
37
|
+
logger: AstroIntegrationLogger,
|
|
38
|
+
config: AstroConfig,
|
|
39
|
+
): Promise<void> {
|
|
40
|
+
const run = () => processPages(dir, pages, options, logger, config)
|
|
41
|
+
return cms ? cms.runAtProjectRoot(fileURLToPath(config.root), run) : run()
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function processPages(
|
|
29
45
|
dir: URL,
|
|
30
46
|
pages: PageInfo[],
|
|
31
47
|
options: ResolvedOptions,
|
|
@@ -33,11 +49,16 @@ export async function processBuildOutput(
|
|
|
33
49
|
config: AstroConfig,
|
|
34
50
|
) {
|
|
35
51
|
const baseUrl = getBaseUrl(config)
|
|
36
|
-
const distDir = dir
|
|
52
|
+
const distDir = fileURLToPath(dir)
|
|
37
53
|
let collectionCount = 0
|
|
38
54
|
let staticCount = 0
|
|
39
55
|
const pageEntries: PageEntry[] = []
|
|
40
56
|
let siteMetadata: SiteMetadata = {}
|
|
57
|
+
// Found once for the whole build: resolving a page against a fresh scan made the build quadratic.
|
|
58
|
+
const site = {
|
|
59
|
+
pages: await findCollectionPages(options.contentDir, message => logger.warn(message)),
|
|
60
|
+
syntax: markdownSyntaxOf(config),
|
|
61
|
+
}
|
|
41
62
|
|
|
42
63
|
for (const page of pages) {
|
|
43
64
|
const pagePath = normalizePath(page.pathname === '' ? '/' : `/${page.pathname}`)
|
|
@@ -46,61 +67,41 @@ export async function processBuildOutput(
|
|
|
46
67
|
const mdPath = getMdOutputPath(distDir, pagePath)
|
|
47
68
|
const htmlPath = getHtmlPath(distDir, pagePath)
|
|
48
69
|
|
|
49
|
-
//
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const output = createCollectionOutput(content.frontmatter, content.body, content.file)
|
|
53
|
-
const markdown = generateMarkdown(output, {
|
|
54
|
-
url: pagePath,
|
|
55
|
-
type: 'collection',
|
|
56
|
-
sourcePath: content.file,
|
|
57
|
-
baseUrl,
|
|
58
|
-
}, options.includeFrontmatter)
|
|
59
|
-
|
|
60
|
-
await writeMarkdownFile(mdPath, markdown)
|
|
61
|
-
await injectLinkIntoHtml(htmlPath, pagePath)
|
|
62
|
-
pageEntries.push({
|
|
63
|
-
pathname: pagePath,
|
|
64
|
-
title: extractTitle(content.frontmatter.title),
|
|
65
|
-
type: 'collection',
|
|
66
|
-
})
|
|
67
|
-
collectionCount++
|
|
68
|
-
continue
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// Fall back to static page handling
|
|
72
|
-
if (!options.includeStaticPages) continue
|
|
73
|
-
|
|
74
|
-
const htmlExists = await fileExists(htmlPath)
|
|
75
|
-
if (!htmlExists) continue
|
|
70
|
+
// Read once, for the markdown and for the link injected into it.
|
|
71
|
+
let html: Promise<string | undefined> | undefined
|
|
72
|
+
const readHtml = () => (html ??= fs.readFile(htmlPath, 'utf-8').catch(() => undefined))
|
|
76
73
|
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
74
|
+
const resolved = await resolvePageMarkdown(pagePath, options, site, async () => {
|
|
75
|
+
const page = await readHtml()
|
|
76
|
+
return page === undefined ? undefined : htmlToMarkdown(page)
|
|
77
|
+
})
|
|
78
|
+
if (!resolved) continue
|
|
80
79
|
|
|
81
|
-
const markdown = generateMarkdown(output, {
|
|
80
|
+
const markdown = generateMarkdown(resolved.output, {
|
|
82
81
|
url: pagePath,
|
|
83
|
-
type:
|
|
82
|
+
type: resolved.type,
|
|
83
|
+
sourcePath: resolved.sourcePath,
|
|
84
84
|
baseUrl,
|
|
85
85
|
}, options.includeFrontmatter)
|
|
86
86
|
|
|
87
87
|
await writeMarkdownFile(mdPath, markdown)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
title: metadata.title,
|
|
92
|
-
type: 'static',
|
|
93
|
-
})
|
|
88
|
+
pageEntries.push({ pathname: pagePath, title: resolved.title, type: resolved.type })
|
|
89
|
+
if (resolved.type === 'collection') collectionCount++
|
|
90
|
+
else staticCount++
|
|
94
91
|
|
|
95
|
-
//
|
|
92
|
+
// The site's title and description, from the rendered homepage, also when an entry tells it.
|
|
96
93
|
if (pagePath === '/') {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
}
|
|
94
|
+
const page = resolved.metadata ? undefined : await readHtml()
|
|
95
|
+
const metadata = resolved.metadata ?? (page === undefined ? undefined : htmlToMarkdown(page).metadata)
|
|
96
|
+
if (metadata) siteMetadata = { title: metadata.title, description: metadata.description }
|
|
101
97
|
}
|
|
102
98
|
|
|
103
|
-
|
|
99
|
+
// The markdown is written whether or not its link can be added to the page.
|
|
100
|
+
try {
|
|
101
|
+
await injectLinkIntoHtml(htmlPath, pagePath, await readHtml())
|
|
102
|
+
} catch (error) {
|
|
103
|
+
logger.warn(`Failed to link ${pagePath} to its markdown: ${error}`)
|
|
104
|
+
}
|
|
104
105
|
} catch (error) {
|
|
105
106
|
logger.warn(`Failed to process ${pagePath}: ${error}`)
|
|
106
107
|
}
|
|
@@ -144,32 +145,13 @@ export async function processBuildOutput(
|
|
|
144
145
|
}
|
|
145
146
|
}
|
|
146
147
|
|
|
147
|
-
async function injectLinkIntoHtml(htmlPath: string, pagePath: string): Promise<void> {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
} catch {
|
|
152
|
-
// File might not exist for some pages
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
async function fileExists(filePath: string): Promise<boolean> {
|
|
157
|
-
try {
|
|
158
|
-
await fs.access(filePath)
|
|
159
|
-
return true
|
|
160
|
-
} catch {
|
|
161
|
-
return false
|
|
162
|
-
}
|
|
148
|
+
async function injectLinkIntoHtml(htmlPath: string, pagePath: string, html: string | undefined): Promise<void> {
|
|
149
|
+
// Some pages have no HTML file
|
|
150
|
+
if (html === undefined) return
|
|
151
|
+
await fs.writeFile(htmlPath, injectMarkdownLink(html, pagePath), 'utf-8')
|
|
163
152
|
}
|
|
164
153
|
|
|
165
154
|
async function writeMarkdownFile(filePath: string, content: string): Promise<void> {
|
|
166
155
|
await fs.mkdir(path.dirname(filePath), { recursive: true })
|
|
167
156
|
await fs.writeFile(filePath, content, 'utf-8')
|
|
168
157
|
}
|
|
169
|
-
|
|
170
|
-
function extractTitle(title: unknown): string | undefined {
|
|
171
|
-
if (typeof title === 'string') {
|
|
172
|
-
return title
|
|
173
|
-
}
|
|
174
|
-
return undefined
|
|
175
|
-
}
|
package/src/cms-marker.ts
CHANGED
|
@@ -1,52 +1,75 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
}
|
|
6
|
-
|
|
7
|
-
export interface ParsedContent {
|
|
8
|
-
frontmatter: Record<string, { value: string; line: number }>
|
|
9
|
-
body: string
|
|
10
|
-
bodyStartLine: number
|
|
11
|
-
file: string
|
|
12
|
-
collectionName: string
|
|
13
|
-
collectionSlug: string
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
type FindCollectionSource = (pagePath: string, contentDir?: string) => Promise<CollectionInfo | undefined>
|
|
17
|
-
type ParseMarkdownContent = (collectionInfo: CollectionInfo) => Promise<ParsedContent | undefined>
|
|
1
|
+
import type { CollectionInfo, MarkdownContent } from '@nuasite/cms'
|
|
2
|
+
import path from 'node:path'
|
|
3
|
+
import { cms } from './cms'
|
|
4
|
+
import { type MarkdownSyntax, mdxBodyAsMarkdown } from './mdx'
|
|
5
|
+
import { normalizePath } from './paths'
|
|
18
6
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
let initialized = false
|
|
7
|
+
/** The pages that render a collection entry, by normalized pathname. */
|
|
8
|
+
export type CollectionPages = Map<string, CollectionInfo>
|
|
22
9
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
10
|
+
/**
|
|
11
|
+
* A page's collection entry. `body` is `undefined` when the entry cannot be told as markdown (an `.mdx`
|
|
12
|
+
* body that renders components or expressions): the page is still the entry's, but its text comes
|
|
13
|
+
* from the rendered HTML.
|
|
14
|
+
*/
|
|
15
|
+
export type CollectionPage = Omit<MarkdownContent, 'body'> & { body: string | undefined }
|
|
26
16
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
// cms-marker not available
|
|
33
|
-
}
|
|
34
|
-
}
|
|
17
|
+
/**
|
|
18
|
+
* Entries whose body is markdown. Any other entry (`.json`, `.yaml`, Markdoc) still owns its page, but
|
|
19
|
+
* is told by the rendered HTML, like an `.mdx` entry that renders components.
|
|
20
|
+
*/
|
|
21
|
+
const MARKDOWN_ENTRY = /\.(md|mdx|markdown)$/i
|
|
35
22
|
|
|
36
23
|
export async function getCollectionContent(
|
|
37
24
|
pagePath: string,
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (!
|
|
43
|
-
|
|
25
|
+
pages: CollectionPages,
|
|
26
|
+
syntax: MarkdownSyntax,
|
|
27
|
+
): Promise<CollectionPage | undefined> {
|
|
28
|
+
const collectionInfo = pages.get(normalizePath(pagePath))
|
|
29
|
+
if (!cms || !collectionInfo) return undefined
|
|
30
|
+
if (!MARKDOWN_ENTRY.test(collectionInfo.file)) {
|
|
31
|
+
return {
|
|
32
|
+
frontmatter: {},
|
|
33
|
+
body: undefined,
|
|
34
|
+
bodyStartLine: 0,
|
|
35
|
+
file: collectionInfo.file,
|
|
36
|
+
collectionName: collectionInfo.name,
|
|
37
|
+
collectionSlug: collectionInfo.slug,
|
|
38
|
+
}
|
|
44
39
|
}
|
|
45
40
|
|
|
46
|
-
const
|
|
47
|
-
if (!
|
|
48
|
-
|
|
41
|
+
const content = await cms.parseMarkdownContent(collectionInfo)
|
|
42
|
+
if (!content) return undefined
|
|
43
|
+
|
|
44
|
+
const body = content.body.replace(/\r\n?/g, '\n')
|
|
45
|
+
return { ...content, body: content.file.toLowerCase().endsWith('.mdx') ? mdxBodyAsMarkdown(body, syntax) : body }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Find the pages that render a collection entry, once, for a caller about to resolve many pages: the URL
|
|
50
|
+
* the CMS editor shows for each entry (see `collectionPagesByPathname` in `@nuasite/cms`). Runs in the
|
|
51
|
+
* current project context. Empty without `@nuasite/cms`, when `contentDir` lies outside the project root
|
|
52
|
+
* (the CMS reads nothing there), or when the scan fails; `warn` reports the last two. Every page is then
|
|
53
|
+
* told by its rendered HTML.
|
|
54
|
+
*/
|
|
55
|
+
export async function findCollectionPages(contentDir: string, warn: (message: string) => void): Promise<CollectionPages> {
|
|
56
|
+
if (!cms) return new Map()
|
|
57
|
+
const root = cms.getProjectRoot()
|
|
58
|
+
const relative = path.relative(root, path.resolve(root, contentDir))
|
|
59
|
+
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
60
|
+
warn(`contentDir ${contentDir} is outside the project root ${root}, every page is converted from its HTML`)
|
|
61
|
+
return new Map()
|
|
49
62
|
}
|
|
50
63
|
|
|
51
|
-
|
|
64
|
+
let found: CollectionPages
|
|
65
|
+
try {
|
|
66
|
+
// Relative, as the CMS spells the source paths it reads entries from.
|
|
67
|
+
found = await cms.collectionPagesByPathname(await cms.scanProjectCollections(relative.split(path.sep).join('/') || '.'))
|
|
68
|
+
} catch (error) {
|
|
69
|
+
warn(`Could not scan the content collections in ${contentDir}, every page is converted from its HTML: ${error}`)
|
|
70
|
+
return new Map()
|
|
71
|
+
}
|
|
72
|
+
const pages: CollectionPages = new Map()
|
|
73
|
+
for (const [pathname, info] of found) pages.set(normalizePath(pathname), info)
|
|
74
|
+
return pages
|
|
52
75
|
}
|
package/src/cms.ts
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type * as Cms from '@nuasite/cms'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `@nuasite/cms`, an optional peer: without it every page is converted from its rendered HTML.
|
|
5
|
+
*
|
|
6
|
+
* Imported here, at the top level, and nowhere later. Astro evaluates an integration through a Vite
|
|
7
|
+
* module runner that it closes once the config is loaded, so an import made in a hook fails in any Astro
|
|
8
|
+
* build, Node or Bun ("Vite module runner has been closed"). A top-level await finishes while the config
|
|
9
|
+
* is still loading.
|
|
10
|
+
*/
|
|
11
|
+
export const cms: typeof Cms | undefined = await import('@nuasite/cms').catch((error: unknown) => {
|
|
12
|
+
if (isMissing(error)) return undefined
|
|
13
|
+
throw error
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
/** The package is not installed, as opposed to installed and failing to load. */
|
|
17
|
+
function isMissing(error: unknown): boolean {
|
|
18
|
+
const code = (error as { code?: unknown } | null)?.code
|
|
19
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
20
|
+
return (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || /Cannot find (module|package)/.test(message))
|
|
21
|
+
// Quoted, so a missing `@nuasite/cms-types` under an installed cms is not taken for cms itself.
|
|
22
|
+
&& /['"]@nuasite\/cms['"]/.test(message)
|
|
23
|
+
}
|
package/src/dev-middleware.ts
CHANGED
|
@@ -1,12 +1,16 @@
|
|
|
1
|
-
import { createPublicStaticFileChecker } from '@nuasite/cms'
|
|
1
|
+
import { createPublicStaticFileChecker } from '@nuasite/cms-core'
|
|
2
2
|
import type { AstroConfig } from 'astro'
|
|
3
|
+
import path from 'node:path'
|
|
3
4
|
import { fileURLToPath } from 'node:url'
|
|
4
5
|
import type { ViteDevServer } from 'vite'
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
6
|
+
import { cms } from './cms'
|
|
7
|
+
import { findCollectionPages } from './cms-marker'
|
|
8
|
+
import { htmlToMarkdown, type HtmlToMarkdownResult } from './html-to-markdown'
|
|
7
9
|
import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
|
|
8
10
|
import { generateLlmsTxt } from './llms-txt-endpoint'
|
|
9
|
-
import {
|
|
11
|
+
import { generateMarkdown } from './markdown-generator'
|
|
12
|
+
import { markdownSyntaxOf } from './mdx'
|
|
13
|
+
import { resolvePageMarkdown, type SiteContent } from './page-markdown'
|
|
10
14
|
import { injectMarkdownLink, LLM_ENDPOINT_PATH, LLMS_TXT_PATH, mdUrlToPagePath, normalizePath } from './paths'
|
|
11
15
|
import type { ResolvedOptions } from './types'
|
|
12
16
|
|
|
@@ -20,86 +24,53 @@ async function generateMarkdownForPath(
|
|
|
20
24
|
host: string,
|
|
21
25
|
options: ResolvedOptions,
|
|
22
26
|
baseUrl: string,
|
|
27
|
+
site: SiteContent,
|
|
23
28
|
): Promise<string | null> {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
if (content) {
|
|
27
|
-
const output = createCollectionOutput(content.frontmatter, content.body, content.file)
|
|
28
|
-
return generateMarkdown(output, {
|
|
29
|
-
url: pagePath,
|
|
30
|
-
type: 'collection',
|
|
31
|
-
sourcePath: content.file,
|
|
32
|
-
baseUrl,
|
|
33
|
-
}, options.includeFrontmatter)
|
|
34
|
-
}
|
|
29
|
+
const resolved = await resolvePageMarkdown(pagePath, options, site, () => fetchPage(host, pagePath))
|
|
30
|
+
if (!resolved) return null
|
|
35
31
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
32
|
+
return generateMarkdown(resolved.output, {
|
|
33
|
+
url: pagePath,
|
|
34
|
+
type: resolved.type,
|
|
35
|
+
sourcePath: resolved.sourcePath,
|
|
36
|
+
baseUrl,
|
|
37
|
+
}, options.includeFrontmatter)
|
|
38
|
+
}
|
|
40
39
|
|
|
40
|
+
/** The dev server's rendering of a page, converted to markdown, or `undefined` when it serves no HTML there. */
|
|
41
|
+
async function fetchPage(host: string, pagePath: string): Promise<HtmlToMarkdownResult | undefined> {
|
|
41
42
|
const response = await fetch(`http://${host}${pagePath}`, {
|
|
42
43
|
headers: { Accept: 'text/html' },
|
|
43
44
|
})
|
|
44
45
|
|
|
45
|
-
if (!response.ok) return
|
|
46
|
+
if (!response.ok) return undefined
|
|
46
47
|
|
|
47
48
|
const contentType = response.headers.get('content-type')
|
|
48
|
-
if (!contentType?.includes('text/html')) return
|
|
49
|
+
if (!contentType?.includes('text/html')) return undefined
|
|
49
50
|
|
|
50
|
-
|
|
51
|
-
const { metadata, body } = htmlToMarkdown(html)
|
|
52
|
-
const output = createStaticOutput(metadata, body)
|
|
53
|
-
|
|
54
|
-
return generateMarkdown(output, {
|
|
55
|
-
url: pagePath,
|
|
56
|
-
type: 'static',
|
|
57
|
-
baseUrl,
|
|
58
|
-
}, options.includeFrontmatter)
|
|
51
|
+
return htmlToMarkdown(await response.text())
|
|
59
52
|
}
|
|
60
53
|
|
|
61
54
|
/**
|
|
62
55
|
* Discover all pages and their metadata for the LLM endpoint
|
|
63
56
|
*/
|
|
64
|
-
async function discoverPages(
|
|
57
|
+
async function discoverPages(
|
|
58
|
+
host: string,
|
|
59
|
+
options: ResolvedOptions,
|
|
60
|
+
site: SiteContent,
|
|
61
|
+
): Promise<{ pages: PageEntry[]; siteMetadata: SiteMetadata }> {
|
|
65
62
|
const pages: PageEntry[] = []
|
|
66
63
|
let siteMetadata: SiteMetadata = {}
|
|
67
64
|
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
})
|
|
74
|
-
if (homeResponse.ok) {
|
|
75
|
-
const html = await homeResponse.text()
|
|
76
|
-
const { metadata } = htmlToMarkdown(html)
|
|
77
|
-
siteMetadata = {
|
|
78
|
-
title: metadata.title,
|
|
79
|
-
description: metadata.description,
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
} catch {
|
|
83
|
-
// Ignore errors
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
// Try to get pages from Astro's dev server manifest via __astro_dev_toolbar__
|
|
87
|
-
// For now, we'll discover pages by checking common routes and the content directory
|
|
88
|
-
// In dev mode, we just report what we can discover
|
|
65
|
+
// For now, we discover only the homepage: the dev server has no list of routes to offer. A homepage
|
|
66
|
+
// the dev server fails to render is still listed when its entry tells it.
|
|
67
|
+
const home = fetchPage(host, '/').catch(() => undefined)
|
|
68
|
+
const metadata = (await home)?.metadata
|
|
69
|
+
if (metadata) siteMetadata = { title: metadata.title, description: metadata.description }
|
|
89
70
|
|
|
90
|
-
// Check if homepage exists
|
|
91
71
|
try {
|
|
92
|
-
const
|
|
93
|
-
if (
|
|
94
|
-
pages.push({ pathname: '/', title: content.frontmatter.title as string | undefined, type: 'collection' })
|
|
95
|
-
} else if (options.includeStaticPages) {
|
|
96
|
-
const response = await fetch(`http://${host}/`, { headers: { Accept: 'text/html' } })
|
|
97
|
-
if (response.ok) {
|
|
98
|
-
const html = await response.text()
|
|
99
|
-
const { metadata } = htmlToMarkdown(html)
|
|
100
|
-
pages.push({ pathname: '/', title: metadata.title, type: 'static' })
|
|
101
|
-
}
|
|
102
|
-
}
|
|
72
|
+
const resolved = await resolvePageMarkdown('/', options, site, () => home)
|
|
73
|
+
if (resolved) pages.push({ pathname: '/', title: resolved.title, type: resolved.type })
|
|
103
74
|
} catch {
|
|
104
75
|
// Ignore
|
|
105
76
|
}
|
|
@@ -121,6 +92,30 @@ function getBaseUrl(config: AstroConfig): string {
|
|
|
121
92
|
*/
|
|
122
93
|
export function createDevMiddleware(server: ViteDevServer, options: ResolvedOptions, config: AstroConfig) {
|
|
123
94
|
const baseUrl = getBaseUrl(config)
|
|
95
|
+
// Entries are read from Astro's root, as in the build, whatever the working directory is. The project
|
|
96
|
+
// and the pages that render an entry are kept between requests, and replaced after a change to the
|
|
97
|
+
// content, the routes or the content config. The routes are the ones the CMS reads, in `src/pages`.
|
|
98
|
+
const root = fileURLToPath(config.root)
|
|
99
|
+
const watched = [path.resolve(root, options.contentDir) + path.sep, path.join(root, 'src', 'pages') + path.sep]
|
|
100
|
+
const syntax = markdownSyntaxOf(config)
|
|
101
|
+
let project = cms?.projectAtRoot(root)
|
|
102
|
+
let site: Promise<SiteContent> | undefined
|
|
103
|
+
server.watcher?.on('all', (_event: string, file: string) => {
|
|
104
|
+
const changed = path.resolve(file)
|
|
105
|
+
if (watched.some(dir => changed.startsWith(dir)) || /content\.config\.[cm]?[jt]s$/.test(changed)) {
|
|
106
|
+
project = cms?.projectAtRoot(root)
|
|
107
|
+
site = undefined
|
|
108
|
+
}
|
|
109
|
+
})
|
|
110
|
+
/** Run `resolve` in the project, against the pages that render an entry. */
|
|
111
|
+
const withSite = <T>(resolve: (site: SiteContent) => Promise<T>): Promise<T> => {
|
|
112
|
+
const run = async () => {
|
|
113
|
+
site ??= findCollectionPages(options.contentDir, message => console.warn(`[llm-enhancements] ${message}`)).then(pages => ({ pages, syntax }))
|
|
114
|
+
return resolve(await site)
|
|
115
|
+
}
|
|
116
|
+
return project ? project.run(run) : run()
|
|
117
|
+
}
|
|
118
|
+
|
|
124
119
|
const isPublicStaticFile = createPublicStaticFileChecker(
|
|
125
120
|
config.publicDir ? fileURLToPath(config.publicDir) : undefined,
|
|
126
121
|
)
|
|
@@ -137,7 +132,7 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
|
|
|
137
132
|
|
|
138
133
|
try {
|
|
139
134
|
const host = req.headers.host || 'localhost:4321'
|
|
140
|
-
const { pages, siteMetadata } = await discoverPages(host, options)
|
|
135
|
+
const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site))
|
|
141
136
|
const content = generateLlmsTxt(pages, { ...siteMetadata, baseUrl }, llmsTxtOptions)
|
|
142
137
|
|
|
143
138
|
res.setHeader('Content-Type', 'text/plain; charset=utf-8')
|
|
@@ -164,7 +159,7 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
|
|
|
164
159
|
|
|
165
160
|
try {
|
|
166
161
|
const host = req.headers.host || 'localhost:4321'
|
|
167
|
-
const { pages, siteMetadata } = await discoverPages(host, options)
|
|
162
|
+
const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site))
|
|
168
163
|
const markdown = generateLlmMarkdown(pages, { ...siteMetadata, baseUrl }, llmEndpointOptions)
|
|
169
164
|
|
|
170
165
|
res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
|
|
@@ -187,11 +182,17 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
|
|
|
187
182
|
return next()
|
|
188
183
|
}
|
|
189
184
|
|
|
190
|
-
|
|
185
|
+
// Decoded, as the pages that render an entry are keyed: `/aktuality/%C5%A1kola.md` is `/aktuality/škola`.
|
|
186
|
+
let pagePath: string
|
|
187
|
+
try {
|
|
188
|
+
pagePath = mdUrlToPagePath(decodeURI(url))
|
|
189
|
+
} catch {
|
|
190
|
+
return next()
|
|
191
|
+
}
|
|
191
192
|
|
|
192
193
|
try {
|
|
193
194
|
const host = req.headers.host || 'localhost:4321'
|
|
194
|
-
const markdown = await generateMarkdownForPath(pagePath, host, options, baseUrl)
|
|
195
|
+
const markdown = await withSite(site => generateMarkdownForPath(pagePath, host, options, baseUrl, site))
|
|
195
196
|
|
|
196
197
|
if (markdown) {
|
|
197
198
|
res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
|
package/src/mdx.ts
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import type { AstroConfig } from 'astro'
|
|
2
|
+
import remarkDirective from 'remark-directive'
|
|
3
|
+
import remarkMdx from 'remark-mdx'
|
|
4
|
+
import remarkParse from 'remark-parse'
|
|
5
|
+
import { unified } from 'unified'
|
|
6
|
+
|
|
7
|
+
const mdx = unified().use(remarkParse).use(remarkMdx)
|
|
8
|
+
const mdxWithDirectives = unified().use(remarkParse).use(remarkMdx).use(remarkDirective)
|
|
9
|
+
|
|
10
|
+
export interface MarkdownSyntax {
|
|
11
|
+
/**
|
|
12
|
+
* The site's markdown pipeline parses directives (`:::list{.dots}`). MDX then reads a `{…}` after a
|
|
13
|
+
* directive as its attributes; without them it is an expression.
|
|
14
|
+
*/
|
|
15
|
+
directives: boolean
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The syntax of the site's markdown pipeline, which `@astrojs/mdx` extends by default: directives when a
|
|
20
|
+
* remark-directive plugin is in it (`@nuasite/nua` adds the CMS's). Plugins given to `mdx()` alone are not
|
|
21
|
+
* visible here.
|
|
22
|
+
*/
|
|
23
|
+
export function markdownSyntaxOf(config: AstroConfig): MarkdownSyntax {
|
|
24
|
+
const plugins = config.markdown?.remarkPlugins ?? []
|
|
25
|
+
const directives = plugins.some(entry => {
|
|
26
|
+
const plugin = Array.isArray(entry) ? entry[0] : entry
|
|
27
|
+
return plugin === 'remark-directive' || (typeof plugin === 'function' && plugin.name === 'remarkDirective')
|
|
28
|
+
})
|
|
29
|
+
return { directives }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Make an `.mdx` entry body readable as plain markdown, or say it cannot be.
|
|
34
|
+
*
|
|
35
|
+
* The body is parsed as MDX, so fences, multi-line ESM and code spans follow the same rules MDX itself
|
|
36
|
+
* uses. ESM (`import`/`export`) is cut out of the source: none of it is content. A body that still renders
|
|
37
|
+
* something the source only names returns `undefined`, so the page's rendered HTML is used instead:
|
|
38
|
+
* a component (`<Card title="…" />`), an expression (`{new Date().getFullYear()}`, `{/* note *\/}`) or a
|
|
39
|
+
* plain HTML tag whose attributes are expressions. A body MDX cannot parse returns `undefined` too.
|
|
40
|
+
* `syntax` says which extensions the site parses MDX with, so the body reads as the site renders it.
|
|
41
|
+
*/
|
|
42
|
+
export function mdxBodyAsMarkdown(body: string, syntax: MarkdownSyntax): string | undefined {
|
|
43
|
+
let tree: Node
|
|
44
|
+
try {
|
|
45
|
+
tree = (syntax.directives ? mdxWithDirectives : mdx).parse(body)
|
|
46
|
+
} catch {
|
|
47
|
+
return undefined
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const esm: Array<[number, number]> = []
|
|
51
|
+
let rendersMore = false
|
|
52
|
+
visit(tree, node => {
|
|
53
|
+
if (node.type === 'mdxjsEsm') {
|
|
54
|
+
const start = node.position?.start.offset
|
|
55
|
+
const end = node.position?.end.offset
|
|
56
|
+
if (start === undefined || end === undefined) rendersMore = true
|
|
57
|
+
else esm.push([start, end])
|
|
58
|
+
} else if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') {
|
|
59
|
+
rendersMore = true
|
|
60
|
+
} else if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
|
|
61
|
+
if (!isPlainHtml(node as JsxElement)) rendersMore = true
|
|
62
|
+
}
|
|
63
|
+
return !rendersMore
|
|
64
|
+
})
|
|
65
|
+
if (rendersMore) return undefined
|
|
66
|
+
|
|
67
|
+
let out = ''
|
|
68
|
+
let from = 0
|
|
69
|
+
for (const [start, end] of esm) {
|
|
70
|
+
out += body.slice(from, start)
|
|
71
|
+
// The blank lines that separated the block go with it.
|
|
72
|
+
from = end
|
|
73
|
+
while (body[from] === '\n' || body[from] === '\r') from++
|
|
74
|
+
}
|
|
75
|
+
out += body.slice(from)
|
|
76
|
+
return out
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface Node {
|
|
80
|
+
type: string
|
|
81
|
+
position?: { start: { offset?: number }; end: { offset?: number } }
|
|
82
|
+
children?: Node[]
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
interface JsxElement extends Node {
|
|
86
|
+
name: string | null
|
|
87
|
+
attributes: Array<{ type: string; value?: unknown }>
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** A lowercase tag (`<div class="note">`) with literal attributes renders as written; a fragment or component does not. */
|
|
91
|
+
function isPlainHtml(node: JsxElement): boolean {
|
|
92
|
+
if (!node.name || !/^[a-z][\w-]*$/.test(node.name)) return false
|
|
93
|
+
return node.attributes.every(attr => attr.type === 'mdxJsxAttribute' && (attr.value === null || typeof attr.value === 'string'))
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Depth-first walk that stops as soon as `enter` returns false. */
|
|
97
|
+
function visit(node: Node, enter: (node: Node) => boolean): boolean {
|
|
98
|
+
if (!enter(node)) return false
|
|
99
|
+
for (const child of node.children ?? []) {
|
|
100
|
+
if (!visit(child, enter)) return false
|
|
101
|
+
}
|
|
102
|
+
return true
|
|
103
|
+
}
|