@nuasite/llm-enhancements 0.59.0 → 0.61.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -14,7 +14,7 @@
14
14
  "directory": "packages/llm-enhancements"
15
15
  },
16
16
  "license": "Apache-2.0",
17
- "version": "0.59.0",
17
+ "version": "0.61.0",
18
18
  "module": "src/index.ts",
19
19
  "types": "src/index.ts",
20
20
  "type": "module",
@@ -26,7 +26,12 @@
26
26
  }
27
27
  },
28
28
  "dependencies": {
29
- "node-html-parser": "^7.1.0"
29
+ "@nuasite/cms-core": "0.61.0",
30
+ "node-html-parser": "^7.1.0",
31
+ "remark-directive": "^3.0.1",
32
+ "remark-mdx": "^3.1.0",
33
+ "remark-parse": "^11.0.0",
34
+ "unified": "^11.0.5"
30
35
  },
31
36
  "devDependencies": {
32
37
  "@types/bun": "1.3.11",
@@ -35,7 +40,7 @@
35
40
  "peerDependencies": {
36
41
  "typescript": "^6.0.2",
37
42
  "vite": "^7.0.0",
38
- "@nuasite/cms": "0.59.0"
43
+ "@nuasite/cms": "0.61.0"
39
44
  },
40
45
  "peerDependenciesMeta": {
41
46
  "@nuasite/cms": {
@@ -1,11 +1,15 @@
1
1
  import type { AstroConfig, AstroIntegrationLogger } from 'astro'
2
2
  import fs from 'node:fs/promises'
3
3
  import path from 'node:path'
4
- import { getCollectionContent } from './cms-marker'
4
+ import { fileURLToPath } from 'node:url'
5
+ import { cms } from './cms'
6
+ import { findCollectionPages } from './cms-marker'
5
7
  import { htmlToMarkdown } from './html-to-markdown'
6
8
  import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
7
9
  import { generateLlmsTxt } from './llms-txt-endpoint'
8
- import { createCollectionOutput, createStaticOutput, generateMarkdown } from './markdown-generator'
10
+ import { generateMarkdown } from './markdown-generator'
11
+ import { markdownSyntaxOf } from './mdx'
12
+ import { resolvePageMarkdown } from './page-markdown'
9
13
  import { getHtmlPath, getLlmOutputPath, getLlmsTxtOutputPath, getMdOutputPath, injectMarkdownLink, normalizePath } from './paths'
10
14
  import type { ResolvedOptions } from './types'
11
15
 
@@ -23,9 +27,21 @@ function getBaseUrl(config: AstroConfig): string {
23
27
  }
24
28
 
25
29
  /**
26
- * Process build output and generate .md files for all pages
30
+ * Process build output and generate .md files for all pages. Entries are read from Astro's project
31
+ * root, which need not be the working directory (`astro build --root site`).
27
32
  */
28
- export async function processBuildOutput(
33
+ export function processBuildOutput(
34
+ dir: URL,
35
+ pages: PageInfo[],
36
+ options: ResolvedOptions,
37
+ logger: AstroIntegrationLogger,
38
+ config: AstroConfig,
39
+ ): Promise<void> {
40
+ const run = () => processPages(dir, pages, options, logger, config)
41
+ return cms ? cms.runAtProjectRoot(fileURLToPath(config.root), run) : run()
42
+ }
43
+
44
+ async function processPages(
29
45
  dir: URL,
30
46
  pages: PageInfo[],
31
47
  options: ResolvedOptions,
@@ -33,11 +49,16 @@ export async function processBuildOutput(
33
49
  config: AstroConfig,
34
50
  ) {
35
51
  const baseUrl = getBaseUrl(config)
36
- const distDir = dir.pathname
52
+ const distDir = fileURLToPath(dir)
37
53
  let collectionCount = 0
38
54
  let staticCount = 0
39
55
  const pageEntries: PageEntry[] = []
40
56
  let siteMetadata: SiteMetadata = {}
57
+ // Found once for the whole build: resolving a page against a fresh scan made the build quadratic.
58
+ const site = {
59
+ pages: await findCollectionPages(options.contentDir, message => logger.warn(message)),
60
+ syntax: markdownSyntaxOf(config),
61
+ }
41
62
 
42
63
  for (const page of pages) {
43
64
  const pagePath = normalizePath(page.pathname === '' ? '/' : `/${page.pathname}`)
@@ -46,61 +67,41 @@ export async function processBuildOutput(
46
67
  const mdPath = getMdOutputPath(distDir, pagePath)
47
68
  const htmlPath = getHtmlPath(distDir, pagePath)
48
69
 
49
- // Try collection page first
50
- const content = await getCollectionContent(pagePath, options.contentDir)
51
- if (content) {
52
- const output = createCollectionOutput(content.frontmatter, content.body, content.file)
53
- const markdown = generateMarkdown(output, {
54
- url: pagePath,
55
- type: 'collection',
56
- sourcePath: content.file,
57
- baseUrl,
58
- }, options.includeFrontmatter)
59
-
60
- await writeMarkdownFile(mdPath, markdown)
61
- await injectLinkIntoHtml(htmlPath, pagePath)
62
- pageEntries.push({
63
- pathname: pagePath,
64
- title: extractTitle(content.frontmatter.title),
65
- type: 'collection',
66
- })
67
- collectionCount++
68
- continue
69
- }
70
-
71
- // Fall back to static page handling
72
- if (!options.includeStaticPages) continue
73
-
74
- const htmlExists = await fileExists(htmlPath)
75
- if (!htmlExists) continue
70
+ // Read once, for the markdown and for the link injected into it.
71
+ let html: Promise<string | undefined> | undefined
72
+ const readHtml = () => (html ??= fs.readFile(htmlPath, 'utf-8').catch(() => undefined))
76
73
 
77
- const html = await fs.readFile(htmlPath, 'utf-8')
78
- const { metadata, body } = htmlToMarkdown(html)
79
- const output = createStaticOutput(metadata, body)
74
+ const resolved = await resolvePageMarkdown(pagePath, options, site, async () => {
75
+ const page = await readHtml()
76
+ return page === undefined ? undefined : htmlToMarkdown(page)
77
+ })
78
+ if (!resolved) continue
80
79
 
81
- const markdown = generateMarkdown(output, {
80
+ const markdown = generateMarkdown(resolved.output, {
82
81
  url: pagePath,
83
- type: 'static',
82
+ type: resolved.type,
83
+ sourcePath: resolved.sourcePath,
84
84
  baseUrl,
85
85
  }, options.includeFrontmatter)
86
86
 
87
87
  await writeMarkdownFile(mdPath, markdown)
88
- await injectLinkIntoHtml(htmlPath, pagePath)
89
- pageEntries.push({
90
- pathname: pagePath,
91
- title: metadata.title,
92
- type: 'static',
93
- })
88
+ pageEntries.push({ pathname: pagePath, title: resolved.title, type: resolved.type })
89
+ if (resolved.type === 'collection') collectionCount++
90
+ else staticCount++
94
91
 
95
- // Extract site metadata from homepage
92
+ // The site's title and description, from the rendered homepage, also when an entry tells it.
96
93
  if (pagePath === '/') {
97
- siteMetadata = {
98
- title: metadata.title,
99
- description: metadata.description,
100
- }
94
+ const page = resolved.metadata ? undefined : await readHtml()
95
+ const metadata = resolved.metadata ?? (page === undefined ? undefined : htmlToMarkdown(page).metadata)
96
+ if (metadata) siteMetadata = { title: metadata.title, description: metadata.description }
101
97
  }
102
98
 
103
- staticCount++
99
+ // The markdown is written whether or not its link can be added to the page.
100
+ try {
101
+ await injectLinkIntoHtml(htmlPath, pagePath, await readHtml())
102
+ } catch (error) {
103
+ logger.warn(`Failed to link ${pagePath} to its markdown: ${error}`)
104
+ }
104
105
  } catch (error) {
105
106
  logger.warn(`Failed to process ${pagePath}: ${error}`)
106
107
  }
@@ -144,32 +145,13 @@ export async function processBuildOutput(
144
145
  }
145
146
  }
146
147
 
147
- async function injectLinkIntoHtml(htmlPath: string, pagePath: string): Promise<void> {
148
- try {
149
- const html = await fs.readFile(htmlPath, 'utf-8')
150
- await fs.writeFile(htmlPath, injectMarkdownLink(html, pagePath), 'utf-8')
151
- } catch {
152
- // File might not exist for some pages
153
- }
154
- }
155
-
156
- async function fileExists(filePath: string): Promise<boolean> {
157
- try {
158
- await fs.access(filePath)
159
- return true
160
- } catch {
161
- return false
162
- }
148
+ async function injectLinkIntoHtml(htmlPath: string, pagePath: string, html: string | undefined): Promise<void> {
149
+ // Some pages have no HTML file
150
+ if (html === undefined) return
151
+ await fs.writeFile(htmlPath, injectMarkdownLink(html, pagePath), 'utf-8')
163
152
  }
164
153
 
165
154
  async function writeMarkdownFile(filePath: string, content: string): Promise<void> {
166
155
  await fs.mkdir(path.dirname(filePath), { recursive: true })
167
156
  await fs.writeFile(filePath, content, 'utf-8')
168
157
  }
169
-
170
- function extractTitle(title: unknown): string | undefined {
171
- if (typeof title === 'string') {
172
- return title
173
- }
174
- return undefined
175
- }
package/src/cms-marker.ts CHANGED
@@ -1,52 +1,75 @@
1
- export interface CollectionInfo {
2
- name: string
3
- slug: string
4
- file: string
5
- }
6
-
7
- export interface ParsedContent {
8
- frontmatter: Record<string, { value: string; line: number }>
9
- body: string
10
- bodyStartLine: number
11
- file: string
12
- collectionName: string
13
- collectionSlug: string
14
- }
15
-
16
- type FindCollectionSource = (pagePath: string, contentDir?: string) => Promise<CollectionInfo | undefined>
17
- type ParseMarkdownContent = (collectionInfo: CollectionInfo) => Promise<ParsedContent | undefined>
1
+ import type { CollectionInfo, MarkdownContent } from '@nuasite/cms'
2
+ import path from 'node:path'
3
+ import { cms } from './cms'
4
+ import { type MarkdownSyntax, mdxBodyAsMarkdown } from './mdx'
5
+ import { normalizePath } from './paths'
18
6
 
19
- let findCollectionSource: FindCollectionSource | undefined
20
- let parseMarkdownContent: ParseMarkdownContent | undefined
21
- let initialized = false
7
+ /** The pages that render a collection entry, by normalized pathname. */
8
+ export type CollectionPages = Map<string, CollectionInfo>
22
9
 
23
- async function init() {
24
- if (initialized) return
25
- initialized = true
10
+ /**
11
+ * A page's collection entry. `body` is `undefined` when the entry cannot be told as markdown (an `.mdx`
12
+ * body that renders components or expressions): the page is still the entry's, but its text comes
13
+ * from the rendered HTML.
14
+ */
15
+ export type CollectionPage = Omit<MarkdownContent, 'body'> & { body: string | undefined }
26
16
 
27
- try {
28
- const cmsMarker = await import('@nuasite/cms')
29
- findCollectionSource = cmsMarker.findCollectionSource
30
- parseMarkdownContent = cmsMarker.parseMarkdownContent
31
- } catch {
32
- // cms-marker not available
33
- }
34
- }
17
+ /**
18
+ * Entries whose body is markdown. Any other entry (`.json`, `.yaml`, Markdoc) still owns its page, but
19
+ * is told by the rendered HTML, like an `.mdx` entry that renders components.
20
+ */
21
+ const MARKDOWN_ENTRY = /\.(md|mdx|markdown)$/i
35
22
 
36
23
  export async function getCollectionContent(
37
24
  pagePath: string,
38
- contentDir: string,
39
- ): Promise<ParsedContent | undefined> {
40
- await init()
41
-
42
- if (!findCollectionSource || !parseMarkdownContent) {
43
- return undefined
25
+ pages: CollectionPages,
26
+ syntax: MarkdownSyntax,
27
+ ): Promise<CollectionPage | undefined> {
28
+ const collectionInfo = pages.get(normalizePath(pagePath))
29
+ if (!cms || !collectionInfo) return undefined
30
+ if (!MARKDOWN_ENTRY.test(collectionInfo.file)) {
31
+ return {
32
+ frontmatter: {},
33
+ body: undefined,
34
+ bodyStartLine: 0,
35
+ file: collectionInfo.file,
36
+ collectionName: collectionInfo.name,
37
+ collectionSlug: collectionInfo.slug,
38
+ }
44
39
  }
45
40
 
46
- const collectionInfo = await findCollectionSource(pagePath, contentDir)
47
- if (!collectionInfo) {
48
- return undefined
41
+ const content = await cms.parseMarkdownContent(collectionInfo)
42
+ if (!content) return undefined
43
+
44
+ const body = content.body.replace(/\r\n?/g, '\n')
45
+ return { ...content, body: content.file.toLowerCase().endsWith('.mdx') ? mdxBodyAsMarkdown(body, syntax) : body }
46
+ }
47
+
48
+ /**
49
+ * Find the pages that render a collection entry, once, for a caller about to resolve many pages: the URL
50
+ * the CMS editor shows for each entry (see `collectionPagesByPathname` in `@nuasite/cms`). Runs in the
51
+ * current project context. Empty without `@nuasite/cms`, when `contentDir` lies outside the project root
52
+ * (the CMS reads nothing there), or when the scan fails; `warn` reports the last two. Every page is then
53
+ * told by its rendered HTML.
54
+ */
55
+ export async function findCollectionPages(contentDir: string, warn: (message: string) => void): Promise<CollectionPages> {
56
+ if (!cms) return new Map()
57
+ const root = cms.getProjectRoot()
58
+ const relative = path.relative(root, path.resolve(root, contentDir))
59
+ if (relative.startsWith('..') || path.isAbsolute(relative)) {
60
+ warn(`contentDir ${contentDir} is outside the project root ${root}, every page is converted from its HTML`)
61
+ return new Map()
49
62
  }
50
63
 
51
- return parseMarkdownContent(collectionInfo)
64
+ let found: CollectionPages
65
+ try {
66
+ // Relative, as the CMS spells the source paths it reads entries from.
67
+ found = await cms.collectionPagesByPathname(await cms.scanProjectCollections(relative.split(path.sep).join('/') || '.'))
68
+ } catch (error) {
69
+ warn(`Could not scan the content collections in ${contentDir}, every page is converted from its HTML: ${error}`)
70
+ return new Map()
71
+ }
72
+ const pages: CollectionPages = new Map()
73
+ for (const [pathname, info] of found) pages.set(normalizePath(pathname), info)
74
+ return pages
52
75
  }
package/src/cms.ts ADDED
@@ -0,0 +1,23 @@
1
+ import type * as Cms from '@nuasite/cms'
2
+
3
+ /**
4
+ * `@nuasite/cms`, an optional peer: without it every page is converted from its rendered HTML.
5
+ *
6
+ * Imported here, at the top level, and nowhere later. Astro evaluates an integration through a Vite
7
+ * module runner that it closes once the config is loaded, so an import made in a hook fails in any Astro
8
+ * build, Node or Bun ("Vite module runner has been closed"). A top-level await finishes while the config
9
+ * is still loading.
10
+ */
11
+ export const cms: typeof Cms | undefined = await import('@nuasite/cms').catch((error: unknown) => {
12
+ if (isMissing(error)) return undefined
13
+ throw error
14
+ })
15
+
16
+ /** The package is not installed, as opposed to installed and failing to load. */
17
+ function isMissing(error: unknown): boolean {
18
+ const code = (error as { code?: unknown } | null)?.code
19
+ const message = error instanceof Error ? error.message : String(error)
20
+ return (code === 'ERR_MODULE_NOT_FOUND' || code === 'MODULE_NOT_FOUND' || /Cannot find (module|package)/.test(message))
21
+ // Quoted, so a missing `@nuasite/cms-types` under an installed cms is not taken for cms itself.
22
+ && /['"]@nuasite\/cms['"]/.test(message)
23
+ }
@@ -1,12 +1,16 @@
1
- import { createPublicStaticFileChecker } from '@nuasite/cms'
1
+ import { createPublicStaticFileChecker } from '@nuasite/cms-core'
2
2
  import type { AstroConfig } from 'astro'
3
+ import path from 'node:path'
3
4
  import { fileURLToPath } from 'node:url'
4
5
  import type { ViteDevServer } from 'vite'
5
- import { getCollectionContent } from './cms-marker'
6
- import { htmlToMarkdown } from './html-to-markdown'
6
+ import { cms } from './cms'
7
+ import { findCollectionPages } from './cms-marker'
8
+ import { htmlToMarkdown, type HtmlToMarkdownResult } from './html-to-markdown'
7
9
  import { generateLlmMarkdown, type PageEntry, type SiteMetadata } from './llm-endpoint'
8
10
  import { generateLlmsTxt } from './llms-txt-endpoint'
9
- import { createCollectionOutput, createStaticOutput, generateMarkdown } from './markdown-generator'
11
+ import { generateMarkdown } from './markdown-generator'
12
+ import { markdownSyntaxOf } from './mdx'
13
+ import { resolvePageMarkdown, type SiteContent } from './page-markdown'
10
14
  import { injectMarkdownLink, LLM_ENDPOINT_PATH, LLMS_TXT_PATH, mdUrlToPagePath, normalizePath } from './paths'
11
15
  import type { ResolvedOptions } from './types'
12
16
 
@@ -20,86 +24,53 @@ async function generateMarkdownForPath(
20
24
  host: string,
21
25
  options: ResolvedOptions,
22
26
  baseUrl: string,
27
+ site: SiteContent,
23
28
  ): Promise<string | null> {
24
- // Try collection page first
25
- const content = await getCollectionContent(pagePath, options.contentDir)
26
- if (content) {
27
- const output = createCollectionOutput(content.frontmatter, content.body, content.file)
28
- return generateMarkdown(output, {
29
- url: pagePath,
30
- type: 'collection',
31
- sourcePath: content.file,
32
- baseUrl,
33
- }, options.includeFrontmatter)
34
- }
29
+ const resolved = await resolvePageMarkdown(pagePath, options, site, () => fetchPage(host, pagePath))
30
+ if (!resolved) return null
35
31
 
36
- // Fall back to static page handling
37
- if (!options.includeStaticPages) {
38
- return null
39
- }
32
+ return generateMarkdown(resolved.output, {
33
+ url: pagePath,
34
+ type: resolved.type,
35
+ sourcePath: resolved.sourcePath,
36
+ baseUrl,
37
+ }, options.includeFrontmatter)
38
+ }
40
39
 
40
+ /** The dev server's rendering of a page, converted to markdown, or `undefined` when it serves no HTML there. */
41
+ async function fetchPage(host: string, pagePath: string): Promise<HtmlToMarkdownResult | undefined> {
41
42
  const response = await fetch(`http://${host}${pagePath}`, {
42
43
  headers: { Accept: 'text/html' },
43
44
  })
44
45
 
45
- if (!response.ok) return null
46
+ if (!response.ok) return undefined
46
47
 
47
48
  const contentType = response.headers.get('content-type')
48
- if (!contentType?.includes('text/html')) return null
49
+ if (!contentType?.includes('text/html')) return undefined
49
50
 
50
- const html = await response.text()
51
- const { metadata, body } = htmlToMarkdown(html)
52
- const output = createStaticOutput(metadata, body)
53
-
54
- return generateMarkdown(output, {
55
- url: pagePath,
56
- type: 'static',
57
- baseUrl,
58
- }, options.includeFrontmatter)
51
+ return htmlToMarkdown(await response.text())
59
52
  }
60
53
 
61
54
  /**
62
55
  * Discover all pages and their metadata for the LLM endpoint
63
56
  */
64
- async function discoverPages(host: string, options: ResolvedOptions): Promise<{ pages: PageEntry[]; siteMetadata: SiteMetadata }> {
57
+ async function discoverPages(
58
+ host: string,
59
+ options: ResolvedOptions,
60
+ site: SiteContent,
61
+ ): Promise<{ pages: PageEntry[]; siteMetadata: SiteMetadata }> {
65
62
  const pages: PageEntry[] = []
66
63
  let siteMetadata: SiteMetadata = {}
67
64
 
68
- // Fetch the sitemap or root to discover pages
69
- // First try to get homepage metadata
70
- try {
71
- const homeResponse = await fetch(`http://${host}/`, {
72
- headers: { Accept: 'text/html' },
73
- })
74
- if (homeResponse.ok) {
75
- const html = await homeResponse.text()
76
- const { metadata } = htmlToMarkdown(html)
77
- siteMetadata = {
78
- title: metadata.title,
79
- description: metadata.description,
80
- }
81
- }
82
- } catch {
83
- // Ignore errors
84
- }
85
-
86
- // Try to get pages from Astro's dev server manifest via __astro_dev_toolbar__
87
- // For now, we'll discover pages by checking common routes and the content directory
88
- // In dev mode, we just report what we can discover
65
+ // For now, we discover only the homepage: the dev server has no list of routes to offer. A homepage
66
+ // the dev server fails to render is still listed when its entry tells it.
67
+ const home = fetchPage(host, '/').catch(() => undefined)
68
+ const metadata = (await home)?.metadata
69
+ if (metadata) siteMetadata = { title: metadata.title, description: metadata.description }
89
70
 
90
- // Check if homepage exists
91
71
  try {
92
- const content = await getCollectionContent('/', options.contentDir)
93
- if (content) {
94
- pages.push({ pathname: '/', title: content.frontmatter.title as string | undefined, type: 'collection' })
95
- } else if (options.includeStaticPages) {
96
- const response = await fetch(`http://${host}/`, { headers: { Accept: 'text/html' } })
97
- if (response.ok) {
98
- const html = await response.text()
99
- const { metadata } = htmlToMarkdown(html)
100
- pages.push({ pathname: '/', title: metadata.title, type: 'static' })
101
- }
102
- }
72
+ const resolved = await resolvePageMarkdown('/', options, site, () => home)
73
+ if (resolved) pages.push({ pathname: '/', title: resolved.title, type: resolved.type })
103
74
  } catch {
104
75
  // Ignore
105
76
  }
@@ -121,6 +92,30 @@ function getBaseUrl(config: AstroConfig): string {
121
92
  */
122
93
  export function createDevMiddleware(server: ViteDevServer, options: ResolvedOptions, config: AstroConfig) {
123
94
  const baseUrl = getBaseUrl(config)
95
+ // Entries are read from Astro's root, as in the build, whatever the working directory is. The project
96
+ // and the pages that render an entry are kept between requests, and replaced after a change to the
97
+ // content, the routes or the content config. The routes are the ones the CMS reads, in `src/pages`.
98
+ const root = fileURLToPath(config.root)
99
+ const watched = [path.resolve(root, options.contentDir) + path.sep, path.join(root, 'src', 'pages') + path.sep]
100
+ const syntax = markdownSyntaxOf(config)
101
+ let project = cms?.projectAtRoot(root)
102
+ let site: Promise<SiteContent> | undefined
103
+ server.watcher?.on('all', (_event: string, file: string) => {
104
+ const changed = path.resolve(file)
105
+ if (watched.some(dir => changed.startsWith(dir)) || /content\.config\.[cm]?[jt]s$/.test(changed)) {
106
+ project = cms?.projectAtRoot(root)
107
+ site = undefined
108
+ }
109
+ })
110
+ /** Run `resolve` in the project, against the pages that render an entry. */
111
+ const withSite = <T>(resolve: (site: SiteContent) => Promise<T>): Promise<T> => {
112
+ const run = async () => {
113
+ site ??= findCollectionPages(options.contentDir, message => console.warn(`[llm-enhancements] ${message}`)).then(pages => ({ pages, syntax }))
114
+ return resolve(await site)
115
+ }
116
+ return project ? project.run(run) : run()
117
+ }
118
+
124
119
  const isPublicStaticFile = createPublicStaticFileChecker(
125
120
  config.publicDir ? fileURLToPath(config.publicDir) : undefined,
126
121
  )
@@ -137,7 +132,7 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
137
132
 
138
133
  try {
139
134
  const host = req.headers.host || 'localhost:4321'
140
- const { pages, siteMetadata } = await discoverPages(host, options)
135
+ const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site))
141
136
  const content = generateLlmsTxt(pages, { ...siteMetadata, baseUrl }, llmsTxtOptions)
142
137
 
143
138
  res.setHeader('Content-Type', 'text/plain; charset=utf-8')
@@ -164,7 +159,7 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
164
159
 
165
160
  try {
166
161
  const host = req.headers.host || 'localhost:4321'
167
- const { pages, siteMetadata } = await discoverPages(host, options)
162
+ const { pages, siteMetadata } = await withSite(site => discoverPages(host, options, site))
168
163
  const markdown = generateLlmMarkdown(pages, { ...siteMetadata, baseUrl }, llmEndpointOptions)
169
164
 
170
165
  res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
@@ -187,11 +182,17 @@ export function createDevMiddleware(server: ViteDevServer, options: ResolvedOpti
187
182
  return next()
188
183
  }
189
184
 
190
- const pagePath = mdUrlToPagePath(url)
185
+ // Decoded, as the pages that render an entry are keyed: `/aktuality/%C5%A1kola.md` is `/aktuality/škola`.
186
+ let pagePath: string
187
+ try {
188
+ pagePath = mdUrlToPagePath(decodeURI(url))
189
+ } catch {
190
+ return next()
191
+ }
191
192
 
192
193
  try {
193
194
  const host = req.headers.host || 'localhost:4321'
194
- const markdown = await generateMarkdownForPath(pagePath, host, options, baseUrl)
195
+ const markdown = await withSite(site => generateMarkdownForPath(pagePath, host, options, baseUrl, site))
195
196
 
196
197
  if (markdown) {
197
198
  res.setHeader('Content-Type', 'text/markdown; charset=utf-8')
package/src/mdx.ts ADDED
@@ -0,0 +1,103 @@
1
+ import type { AstroConfig } from 'astro'
2
+ import remarkDirective from 'remark-directive'
3
+ import remarkMdx from 'remark-mdx'
4
+ import remarkParse from 'remark-parse'
5
+ import { unified } from 'unified'
6
+
7
+ const mdx = unified().use(remarkParse).use(remarkMdx)
8
+ const mdxWithDirectives = unified().use(remarkParse).use(remarkMdx).use(remarkDirective)
9
+
10
+ export interface MarkdownSyntax {
11
+ /**
12
+ * The site's markdown pipeline parses directives (`:::list{.dots}`). MDX then reads a `{…}` after a
13
+ * directive as its attributes; without them it is an expression.
14
+ */
15
+ directives: boolean
16
+ }
17
+
18
+ /**
19
+ * The syntax of the site's markdown pipeline, which `@astrojs/mdx` extends by default: directives when a
20
+ * remark-directive plugin is in it (`@nuasite/nua` adds the CMS's). Plugins given to `mdx()` alone are not
21
+ * visible here.
22
+ */
23
+ export function markdownSyntaxOf(config: AstroConfig): MarkdownSyntax {
24
+ const plugins = config.markdown?.remarkPlugins ?? []
25
+ const directives = plugins.some(entry => {
26
+ const plugin = Array.isArray(entry) ? entry[0] : entry
27
+ return plugin === 'remark-directive' || (typeof plugin === 'function' && plugin.name === 'remarkDirective')
28
+ })
29
+ return { directives }
30
+ }
31
+
32
+ /**
33
+ * Make an `.mdx` entry body readable as plain markdown, or say it cannot be.
34
+ *
35
+ * The body is parsed as MDX, so fences, multi-line ESM and code spans follow the same rules MDX itself
36
+ * uses. ESM (`import`/`export`) is cut out of the source: none of it is content. A body that still renders
37
+ * something the source only names returns `undefined`, so the page's rendered HTML is used instead:
38
+ * a component (`<Card title="…" />`), an expression (`{new Date().getFullYear()}`, `{/* note *\/}`) or a
39
+ * plain HTML tag whose attributes are expressions. A body MDX cannot parse returns `undefined` too.
40
+ * `syntax` says which extensions the site parses MDX with, so the body reads as the site renders it.
41
+ */
42
+ export function mdxBodyAsMarkdown(body: string, syntax: MarkdownSyntax): string | undefined {
43
+ let tree: Node
44
+ try {
45
+ tree = (syntax.directives ? mdxWithDirectives : mdx).parse(body)
46
+ } catch {
47
+ return undefined
48
+ }
49
+
50
+ const esm: Array<[number, number]> = []
51
+ let rendersMore = false
52
+ visit(tree, node => {
53
+ if (node.type === 'mdxjsEsm') {
54
+ const start = node.position?.start.offset
55
+ const end = node.position?.end.offset
56
+ if (start === undefined || end === undefined) rendersMore = true
57
+ else esm.push([start, end])
58
+ } else if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') {
59
+ rendersMore = true
60
+ } else if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
61
+ if (!isPlainHtml(node as JsxElement)) rendersMore = true
62
+ }
63
+ return !rendersMore
64
+ })
65
+ if (rendersMore) return undefined
66
+
67
+ let out = ''
68
+ let from = 0
69
+ for (const [start, end] of esm) {
70
+ out += body.slice(from, start)
71
+ // The blank lines that separated the block go with it.
72
+ from = end
73
+ while (body[from] === '\n' || body[from] === '\r') from++
74
+ }
75
+ out += body.slice(from)
76
+ return out
77
+ }
78
+
79
+ interface Node {
80
+ type: string
81
+ position?: { start: { offset?: number }; end: { offset?: number } }
82
+ children?: Node[]
83
+ }
84
+
85
+ interface JsxElement extends Node {
86
+ name: string | null
87
+ attributes: Array<{ type: string; value?: unknown }>
88
+ }
89
+
90
+ /** A lowercase tag (`<div class="note">`) with literal attributes renders as written; a fragment or component does not. */
91
+ function isPlainHtml(node: JsxElement): boolean {
92
+ if (!node.name || !/^[a-z][\w-]*$/.test(node.name)) return false
93
+ return node.attributes.every(attr => attr.type === 'mdxJsxAttribute' && (attr.value === null || typeof attr.value === 'string'))
94
+ }
95
+
96
+ /** Depth-first walk that stops as soon as `enter` returns false. */
97
+ function visit(node: Node, enter: (node: Node) => boolean): boolean {
98
+ if (!enter(node)) return false
99
+ for (const child of node.children ?? []) {
100
+ if (!visit(child, enter)) return false
101
+ }
102
+ return true
103
+ }