@nuasite/llm-enhancements 0.64.3 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{types/build-processor.d.ts → build-processor.d.ts} +1 -1
- package/dist/build-processor.d.ts.map +1 -0
- package/dist/build-processor.js +135 -0
- package/dist/{types/cms-marker.d.ts → cms-marker.d.ts} +1 -1
- package/dist/cms-marker.d.ts.map +1 -0
- package/dist/cms-marker.js +59 -0
- package/dist/cms.d.ts.map +1 -0
- package/dist/cms.js +21 -0
- package/dist/{types/dev-middleware.d.ts → dev-middleware.d.ts} +1 -1
- package/dist/dev-middleware.d.ts.map +1 -0
- package/dist/dev-middleware.js +216 -0
- package/dist/html-to-markdown.d.ts.map +1 -0
- package/dist/html-to-markdown.js +298 -0
- package/dist/{types/index.d.ts → index.d.ts} +2 -2
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +23 -0
- package/dist/{types/llm-endpoint.d.ts → llm-endpoint.d.ts} +1 -1
- package/dist/llm-endpoint.d.ts.map +1 -0
- package/dist/llm-endpoint.js +52 -0
- package/dist/{types/llms-txt-endpoint.d.ts → llms-txt-endpoint.d.ts} +1 -1
- package/dist/llms-txt-endpoint.d.ts.map +1 -0
- package/dist/llms-txt-endpoint.js +92 -0
- package/dist/{types/markdown-generator.d.ts → markdown-generator.d.ts} +1 -1
- package/dist/markdown-generator.d.ts.map +1 -0
- package/dist/markdown-generator.js +106 -0
- package/dist/mdx.d.ts.map +1 -0
- package/dist/mdx.js +90 -0
- package/dist/{types/page-markdown.d.ts → page-markdown.d.ts} +4 -4
- package/dist/page-markdown.d.ts.map +1 -0
- package/dist/page-markdown.js +34 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +79 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/dist/types/tests/tsconfig.tsbuildinfo +1 -1
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +11 -0
- package/package.json +9 -9
- package/src/build-processor.ts +10 -10
- package/src/cms-marker.ts +3 -3
- package/src/dev-middleware.ts +10 -10
- package/src/index.ts +4 -4
- package/src/llm-endpoint.ts +2 -2
- package/src/llms-txt-endpoint.ts +2 -2
- package/src/markdown-generator.ts +1 -1
- package/src/page-markdown.ts +5 -5
- package/src/tsconfig.json +5 -1
- package/dist/types/build-processor.d.ts.map +0 -1
- package/dist/types/cms-marker.d.ts.map +0 -1
- package/dist/types/cms.d.ts.map +0 -1
- package/dist/types/dev-middleware.d.ts.map +0 -1
- package/dist/types/html-to-markdown.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/llm-endpoint.d.ts.map +0 -1
- package/dist/types/llms-txt-endpoint.d.ts.map +0 -1
- package/dist/types/markdown-generator.d.ts.map +0 -1
- package/dist/types/mdx.d.ts.map +0 -1
- package/dist/types/page-markdown.d.ts.map +0 -1
- package/dist/types/paths.d.ts.map +0 -1
- package/dist/types/tsconfig.tsbuildinfo +0 -1
- package/dist/types/types.d.ts.map +0 -1
- /package/dist/{types/cms.d.ts → cms.d.ts} +0 -0
- /package/dist/{types/html-to-markdown.d.ts → html-to-markdown.d.ts} +0 -0
- /package/dist/{types/mdx.d.ts → mdx.d.ts} +0 -0
- /package/dist/{types/paths.d.ts → paths.d.ts} +0 -0
- /package/dist/{types/types.d.ts → types.d.ts} +0 -0
package/dist/mdx.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import remarkDirective from 'remark-directive';
|
|
2
|
+
import remarkMdx from 'remark-mdx';
|
|
3
|
+
import remarkParse from 'remark-parse';
|
|
4
|
+
import { unified } from 'unified';
|
|
5
|
+
const mdx = unified().use(remarkParse).use(remarkMdx);
|
|
6
|
+
const mdxWithDirectives = unified().use(remarkParse).use(remarkMdx).use(remarkDirective);
|
|
7
|
+
/**
|
|
8
|
+
* The syntax of the site's markdown pipeline, which `@astrojs/mdx` extends by default: directives when a
|
|
9
|
+
* remark-directive plugin is in it (`@nuasite/nua` adds the CMS's). Plugins given to `mdx()` alone are not
|
|
10
|
+
* visible here. Since Astro 7 the plugins sit on the `unified()` processor; the deprecated top-level
|
|
11
|
+
* `markdown.remarkPlugins` is read too.
|
|
12
|
+
*/
|
|
13
|
+
export function markdownSyntaxOf(config) {
|
|
14
|
+
const processor = config.markdown?.processor;
|
|
15
|
+
const processorPlugins = processor?.name === 'unified' ? processor.options.remarkPlugins : undefined;
|
|
16
|
+
const plugins = [...(config.markdown?.remarkPlugins ?? []), ...(processorPlugins ?? [])];
|
|
17
|
+
const directives = plugins.some(entry => {
|
|
18
|
+
const plugin = Array.isArray(entry) ? entry[0] : entry;
|
|
19
|
+
return plugin === 'remark-directive' || (typeof plugin === 'function' && plugin.name === 'remarkDirective');
|
|
20
|
+
});
|
|
21
|
+
return { directives };
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Make an `.mdx` entry body readable as plain markdown, or say it cannot be.
|
|
25
|
+
*
|
|
26
|
+
* The body is parsed as MDX, so fences, multi-line ESM and code spans follow the same rules MDX itself
|
|
27
|
+
* uses. ESM (`import`/`export`) is cut out of the source: none of it is content. A body that still renders
|
|
28
|
+
* something the source only names returns `undefined`, so the page's rendered HTML is used instead:
|
|
29
|
+
* a component (`<Card title="…" />`), an expression (`{new Date().getFullYear()}`, `{/* note *\/}`) or a
|
|
30
|
+
* plain HTML tag whose attributes are expressions. A body MDX cannot parse returns `undefined` too.
|
|
31
|
+
* `syntax` says which extensions the site parses MDX with, so the body reads as the site renders it.
|
|
32
|
+
*/
|
|
33
|
+
export function mdxBodyAsMarkdown(body, syntax) {
|
|
34
|
+
let tree;
|
|
35
|
+
try {
|
|
36
|
+
tree = (syntax.directives ? mdxWithDirectives : mdx).parse(body);
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return undefined;
|
|
40
|
+
}
|
|
41
|
+
const esm = [];
|
|
42
|
+
let rendersMore = false;
|
|
43
|
+
visit(tree, node => {
|
|
44
|
+
if (node.type === 'mdxjsEsm') {
|
|
45
|
+
const start = node.position?.start.offset;
|
|
46
|
+
const end = node.position?.end.offset;
|
|
47
|
+
if (start === undefined || end === undefined)
|
|
48
|
+
rendersMore = true;
|
|
49
|
+
else
|
|
50
|
+
esm.push([start, end]);
|
|
51
|
+
}
|
|
52
|
+
else if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') {
|
|
53
|
+
rendersMore = true;
|
|
54
|
+
}
|
|
55
|
+
else if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
|
|
56
|
+
if (!isPlainHtml(node))
|
|
57
|
+
rendersMore = true;
|
|
58
|
+
}
|
|
59
|
+
return !rendersMore;
|
|
60
|
+
});
|
|
61
|
+
if (rendersMore)
|
|
62
|
+
return undefined;
|
|
63
|
+
let out = '';
|
|
64
|
+
let from = 0;
|
|
65
|
+
for (const [start, end] of esm) {
|
|
66
|
+
out += body.slice(from, start);
|
|
67
|
+
// The blank lines that separated the block go with it.
|
|
68
|
+
from = end;
|
|
69
|
+
while (body[from] === '\n' || body[from] === '\r')
|
|
70
|
+
from++;
|
|
71
|
+
}
|
|
72
|
+
out += body.slice(from);
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
/** A lowercase tag (`<div class="note">`) with literal attributes renders as written; a fragment or component does not. */
|
|
76
|
+
function isPlainHtml(node) {
|
|
77
|
+
if (!node.name || !/^[a-z][\w-]*$/.test(node.name))
|
|
78
|
+
return false;
|
|
79
|
+
return node.attributes.every(attr => attr.type === 'mdxJsxAttribute' && (attr.value === null || typeof attr.value === 'string'));
|
|
80
|
+
}
|
|
81
|
+
/** Depth-first walk that stops as soon as `enter` returns false. */
|
|
82
|
+
function visit(node, enter) {
|
|
83
|
+
if (!enter(node))
|
|
84
|
+
return false;
|
|
85
|
+
for (const child of node.children ?? []) {
|
|
86
|
+
if (!visit(child, enter))
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { type CollectionPages } from './cms-marker';
|
|
2
|
-
import type { HtmlToMarkdownResult } from './html-to-markdown';
|
|
3
|
-
import type { MarkdownSyntax } from './mdx';
|
|
4
|
-
import type { MarkdownOutput, ResolvedOptions } from './types';
|
|
1
|
+
import { type CollectionPages } from './cms-marker.ts';
|
|
2
|
+
import type { HtmlToMarkdownResult } from './html-to-markdown.ts';
|
|
3
|
+
import type { MarkdownSyntax } from './mdx.ts';
|
|
4
|
+
import type { MarkdownOutput, ResolvedOptions } from './types.ts';
|
|
5
5
|
/** What a page is resolved against: the pages that render an entry, and how the site parses markdown. */
|
|
6
6
|
export interface SiteContent {
|
|
7
7
|
pages: CollectionPages;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"page-markdown.d.ts","sourceRoot":"","sources":["../src/page-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAwB,MAAM,iBAAiB,CAAA;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAEjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjE,yGAAyG;AACzG,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,eAAe,CAAA;IACtB,MAAM,EAAE,cAAc,CAAA;CACtB;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,cAAc,CAAA;IACtB,IAAI,EAAE,YAAY,GAAG,QAAQ,CAAA;IAC7B,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAA;CAC3C;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACxC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,WAAW,EACjB,QAAQ,EAAE,MAAM,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAyBnC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { getCollectionContent } from "./cms-marker.js";
|
|
2
|
+
import { createCollectionOutput, createStaticOutput } from "./markdown-generator.js";
|
|
3
|
+
/**
|
|
4
|
+
* Resolve a page to the markdown that tells it: its collection entry when it has one, else its rendered
|
|
5
|
+
* HTML when static pages are wanted. `readPage` converts the rendered page, or gives `undefined` when
|
|
6
|
+
* there is none; it is called at most once and only when the page is needed.
|
|
7
|
+
*/
|
|
8
|
+
export async function resolvePageMarkdown(pagePath, options, site, readPage) {
|
|
9
|
+
let page;
|
|
10
|
+
const renderedPage = () => (page ??= readPage());
|
|
11
|
+
const entry = await getCollectionContent(pagePath, site.pages, site.syntax);
|
|
12
|
+
if (entry) {
|
|
13
|
+
const entryTitle = entry.frontmatter.title?.value;
|
|
14
|
+
// An entry that renders more than its source says is told by its page, which is still the entry's,
|
|
15
|
+
// so it is written whether or not static pages are.
|
|
16
|
+
const rendered = entry.body === undefined || entryTitle === undefined ? await renderedPage() : undefined;
|
|
17
|
+
const body = entry.body ?? rendered?.body;
|
|
18
|
+
if (body === undefined)
|
|
19
|
+
return undefined;
|
|
20
|
+
return {
|
|
21
|
+
output: createCollectionOutput(entry.frontmatter, body, entry.file),
|
|
22
|
+
type: 'collection',
|
|
23
|
+
sourcePath: entry.file,
|
|
24
|
+
title: entryTitle ?? rendered?.metadata.title,
|
|
25
|
+
metadata: rendered?.metadata,
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (!options.includeStaticPages)
|
|
29
|
+
return undefined;
|
|
30
|
+
const rendered = await renderedPage();
|
|
31
|
+
if (!rendered)
|
|
32
|
+
return undefined;
|
|
33
|
+
return { output: createStaticOutput(rendered.metadata, rendered.body), type: 'static', title: rendered.metadata.title, metadata: rendered.metadata };
|
|
34
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAEA,iDAAiD;AACjD,eAAO,MAAM,iBAAiB,wBAAwB,CAAA;AAEtD,iCAAiC;AACjC,eAAO,MAAM,aAAa,cAAc,CAAA;AAExC;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMjD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMvD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMnD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKrE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAWzE"}
|
package/dist/paths.js
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
/** Well-known path for LLM discovery endpoint */
|
|
3
|
+
export const LLM_ENDPOINT_PATH = '/.well-known/llm.md';
|
|
4
|
+
/** Path for llms.txt endpoint */
|
|
5
|
+
export const LLMS_TXT_PATH = '/llms.txt';
|
|
6
|
+
/**
|
|
7
|
+
* Normalize a URL path by removing query strings, hashes, and trailing slashes
|
|
8
|
+
*/
|
|
9
|
+
export function normalizePath(url) {
|
|
10
|
+
let pagePath = url.split('?')[0]?.split('#')[0] ?? '';
|
|
11
|
+
if (pagePath.length > 1 && pagePath.endsWith('/')) {
|
|
12
|
+
pagePath = pagePath.slice(0, -1);
|
|
13
|
+
}
|
|
14
|
+
return pagePath || '/';
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Get the markdown URL for a given page path
|
|
18
|
+
*/
|
|
19
|
+
export function getMarkdownUrl(pagePath) {
|
|
20
|
+
if (pagePath === '/') {
|
|
21
|
+
return '/index.md';
|
|
22
|
+
}
|
|
23
|
+
const cleanPath = pagePath.endsWith('/') ? pagePath.slice(0, -1) : pagePath;
|
|
24
|
+
return `${cleanPath}.md`;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Convert a .md URL back to a page path
|
|
28
|
+
*/
|
|
29
|
+
export function mdUrlToPagePath(url) {
|
|
30
|
+
const match = url.match(/^\/(.*)\.md$/);
|
|
31
|
+
if (!match)
|
|
32
|
+
return url;
|
|
33
|
+
const pagePath = '/' + match[1];
|
|
34
|
+
return pagePath === '/index' ? '/' : pagePath;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Get the output path for a .md file in dist
|
|
38
|
+
*/
|
|
39
|
+
export function getMdOutputPath(distDir, pagePath) {
|
|
40
|
+
if (pagePath === '/') {
|
|
41
|
+
return path.join(distDir, 'index.md');
|
|
42
|
+
}
|
|
43
|
+
return path.join(distDir, `${pagePath.slice(1)}.md`);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Get the HTML file path for a page in dist
|
|
47
|
+
*/
|
|
48
|
+
export function getHtmlPath(distDir, pagePath) {
|
|
49
|
+
if (pagePath === '/') {
|
|
50
|
+
return path.join(distDir, 'index.html');
|
|
51
|
+
}
|
|
52
|
+
return path.join(distDir, pagePath.slice(1), 'index.html');
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Get the output path for llm.md in dist
|
|
56
|
+
*/
|
|
57
|
+
export function getLlmOutputPath(distDir) {
|
|
58
|
+
return path.join(distDir, '.well-known', 'llm.md');
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Get the output path for llms.txt in dist
|
|
62
|
+
*/
|
|
63
|
+
export function getLlmsTxtOutputPath(distDir) {
|
|
64
|
+
return path.join(distDir, 'llms.txt');
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Inject markdown alternate link into HTML head
|
|
68
|
+
*/
|
|
69
|
+
export function injectMarkdownLink(html, pagePath) {
|
|
70
|
+
const mdUrl = getMarkdownUrl(pagePath);
|
|
71
|
+
const linkTag = `<link rel="alternate" type="text/markdown" href="${mdUrl}">`;
|
|
72
|
+
if (html.includes('</head>')) {
|
|
73
|
+
return html.replace('</head>', `${linkTag}\n</head>`);
|
|
74
|
+
}
|
|
75
|
+
if (html.includes('<head>')) {
|
|
76
|
+
return html.replace('<head>', `<head>\n${linkTag}`);
|
|
77
|
+
}
|
|
78
|
+
return html;
|
|
79
|
+
}
|