@nuasite/llm-enhancements 0.64.3 → 0.65.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/{types/build-processor.d.ts → build-processor.d.ts} +1 -1
  2. package/dist/build-processor.d.ts.map +1 -0
  3. package/dist/build-processor.js +135 -0
  4. package/dist/{types/cms-marker.d.ts → cms-marker.d.ts} +1 -1
  5. package/dist/cms-marker.d.ts.map +1 -0
  6. package/dist/cms-marker.js +59 -0
  7. package/dist/cms.d.ts.map +1 -0
  8. package/dist/cms.js +21 -0
  9. package/dist/{types/dev-middleware.d.ts → dev-middleware.d.ts} +1 -1
  10. package/dist/dev-middleware.d.ts.map +1 -0
  11. package/dist/dev-middleware.js +216 -0
  12. package/dist/html-to-markdown.d.ts.map +1 -0
  13. package/dist/html-to-markdown.js +298 -0
  14. package/dist/{types/index.d.ts → index.d.ts} +2 -2
  15. package/dist/index.d.ts.map +1 -0
  16. package/dist/index.js +23 -0
  17. package/dist/{types/llm-endpoint.d.ts → llm-endpoint.d.ts} +1 -1
  18. package/dist/llm-endpoint.d.ts.map +1 -0
  19. package/dist/llm-endpoint.js +52 -0
  20. package/dist/{types/llms-txt-endpoint.d.ts → llms-txt-endpoint.d.ts} +1 -1
  21. package/dist/llms-txt-endpoint.d.ts.map +1 -0
  22. package/dist/llms-txt-endpoint.js +92 -0
  23. package/dist/{types/markdown-generator.d.ts → markdown-generator.d.ts} +1 -1
  24. package/dist/markdown-generator.d.ts.map +1 -0
  25. package/dist/markdown-generator.js +106 -0
  26. package/dist/mdx.d.ts.map +1 -0
  27. package/dist/mdx.js +90 -0
  28. package/dist/{types/page-markdown.d.ts → page-markdown.d.ts} +4 -4
  29. package/dist/page-markdown.d.ts.map +1 -0
  30. package/dist/page-markdown.js +34 -0
  31. package/dist/paths.d.ts.map +1 -0
  32. package/dist/paths.js +79 -0
  33. package/dist/tsconfig.tsbuildinfo +1 -0
  34. package/dist/types/tests/tsconfig.tsbuildinfo +1 -1
  35. package/dist/types.d.ts.map +1 -0
  36. package/dist/types.js +11 -0
  37. package/package.json +10 -10
  38. package/src/build-processor.ts +10 -10
  39. package/src/cms-marker.ts +3 -3
  40. package/src/dev-middleware.ts +10 -10
  41. package/src/index.ts +4 -4
  42. package/src/llm-endpoint.ts +2 -2
  43. package/src/llms-txt-endpoint.ts +2 -2
  44. package/src/markdown-generator.ts +1 -1
  45. package/src/page-markdown.ts +5 -5
  46. package/src/tsconfig.json +5 -1
  47. package/dist/types/build-processor.d.ts.map +0 -1
  48. package/dist/types/cms-marker.d.ts.map +0 -1
  49. package/dist/types/cms.d.ts.map +0 -1
  50. package/dist/types/dev-middleware.d.ts.map +0 -1
  51. package/dist/types/html-to-markdown.d.ts.map +0 -1
  52. package/dist/types/index.d.ts.map +0 -1
  53. package/dist/types/llm-endpoint.d.ts.map +0 -1
  54. package/dist/types/llms-txt-endpoint.d.ts.map +0 -1
  55. package/dist/types/markdown-generator.d.ts.map +0 -1
  56. package/dist/types/mdx.d.ts.map +0 -1
  57. package/dist/types/page-markdown.d.ts.map +0 -1
  58. package/dist/types/paths.d.ts.map +0 -1
  59. package/dist/types/tsconfig.tsbuildinfo +0 -1
  60. package/dist/types/types.d.ts.map +0 -1
  61. /package/dist/{types/cms.d.ts → cms.d.ts} +0 -0
  62. /package/dist/{types/html-to-markdown.d.ts → html-to-markdown.d.ts} +0 -0
  63. /package/dist/{types/mdx.d.ts → mdx.d.ts} +0 -0
  64. /package/dist/{types/paths.d.ts → paths.d.ts} +0 -0
  65. /package/dist/{types/types.d.ts → types.d.ts} +0 -0
package/dist/mdx.js ADDED
@@ -0,0 +1,90 @@
1
+ import remarkDirective from 'remark-directive';
2
+ import remarkMdx from 'remark-mdx';
3
+ import remarkParse from 'remark-parse';
4
+ import { unified } from 'unified';
5
+ const mdx = unified().use(remarkParse).use(remarkMdx);
6
+ const mdxWithDirectives = unified().use(remarkParse).use(remarkMdx).use(remarkDirective);
7
+ /**
8
+ * The syntax of the site's markdown pipeline, which `@astrojs/mdx` extends by default: directives when a
9
+ * remark-directive plugin is in it (`@nuasite/nua` adds the CMS's). Plugins given to `mdx()` alone are not
10
+ * visible here. Since Astro 7 the plugins sit on the `unified()` processor; the deprecated top-level
11
+ * `markdown.remarkPlugins` is read too.
12
+ */
13
+ export function markdownSyntaxOf(config) {
14
+ const processor = config.markdown?.processor;
15
+ const processorPlugins = processor?.name === 'unified' ? processor.options.remarkPlugins : undefined;
16
+ const plugins = [...(config.markdown?.remarkPlugins ?? []), ...(processorPlugins ?? [])];
17
+ const directives = plugins.some(entry => {
18
+ const plugin = Array.isArray(entry) ? entry[0] : entry;
19
+ return plugin === 'remark-directive' || (typeof plugin === 'function' && plugin.name === 'remarkDirective');
20
+ });
21
+ return { directives };
22
+ }
23
+ /**
24
+ * Make an `.mdx` entry body readable as plain markdown, or say it cannot be.
25
+ *
26
+ * The body is parsed as MDX, so fences, multi-line ESM and code spans follow the same rules MDX itself
27
+ * uses. ESM (`import`/`export`) is cut out of the source: none of it is content. A body that still renders
28
+ * something the source only names returns `undefined`, so the page's rendered HTML is used instead:
29
+ * a component (`<Card title="…" />`), an expression (`{new Date().getFullYear()}`, `{/* note *\/}`) or a
30
+ * plain HTML tag whose attributes are expressions. A body MDX cannot parse returns `undefined` too.
31
+ * `syntax` says which extensions the site parses MDX with, so the body reads as the site renders it.
32
+ */
33
+ export function mdxBodyAsMarkdown(body, syntax) {
34
+ let tree;
35
+ try {
36
+ tree = (syntax.directives ? mdxWithDirectives : mdx).parse(body);
37
+ }
38
+ catch {
39
+ return undefined;
40
+ }
41
+ const esm = [];
42
+ let rendersMore = false;
43
+ visit(tree, node => {
44
+ if (node.type === 'mdxjsEsm') {
45
+ const start = node.position?.start.offset;
46
+ const end = node.position?.end.offset;
47
+ if (start === undefined || end === undefined)
48
+ rendersMore = true;
49
+ else
50
+ esm.push([start, end]);
51
+ }
52
+ else if (node.type === 'mdxFlowExpression' || node.type === 'mdxTextExpression') {
53
+ rendersMore = true;
54
+ }
55
+ else if (node.type === 'mdxJsxFlowElement' || node.type === 'mdxJsxTextElement') {
56
+ if (!isPlainHtml(node))
57
+ rendersMore = true;
58
+ }
59
+ return !rendersMore;
60
+ });
61
+ if (rendersMore)
62
+ return undefined;
63
+ let out = '';
64
+ let from = 0;
65
+ for (const [start, end] of esm) {
66
+ out += body.slice(from, start);
67
+ // The blank lines that separated the block go with it.
68
+ from = end;
69
+ while (body[from] === '\n' || body[from] === '\r')
70
+ from++;
71
+ }
72
+ out += body.slice(from);
73
+ return out;
74
+ }
75
+ /** A lowercase tag (`<div class="note">`) with literal attributes renders as written; a fragment or component does not. */
76
+ function isPlainHtml(node) {
77
+ if (!node.name || !/^[a-z][\w-]*$/.test(node.name))
78
+ return false;
79
+ return node.attributes.every(attr => attr.type === 'mdxJsxAttribute' && (attr.value === null || typeof attr.value === 'string'));
80
+ }
81
+ /** Depth-first walk that stops as soon as `enter` returns false. */
82
+ function visit(node, enter) {
83
+ if (!enter(node))
84
+ return false;
85
+ for (const child of node.children ?? []) {
86
+ if (!visit(child, enter))
87
+ return false;
88
+ }
89
+ return true;
90
+ }
@@ -1,7 +1,7 @@
1
- import { type CollectionPages } from './cms-marker';
2
- import type { HtmlToMarkdownResult } from './html-to-markdown';
3
- import type { MarkdownSyntax } from './mdx';
4
- import type { MarkdownOutput, ResolvedOptions } from './types';
1
+ import { type CollectionPages } from './cms-marker.ts';
2
+ import type { HtmlToMarkdownResult } from './html-to-markdown.ts';
3
+ import type { MarkdownSyntax } from './mdx.ts';
4
+ import type { MarkdownOutput, ResolvedOptions } from './types.ts';
5
5
  /** What a page is resolved against: the pages that render an entry, and how the site parses markdown. */
6
6
  export interface SiteContent {
7
7
  pages: CollectionPages;
@@ -0,0 +1 @@
1
+ {"version":3,"file":"page-markdown.d.ts","sourceRoot":"","sources":["../src/page-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,eAAe,EAAwB,MAAM,iBAAiB,CAAA;AAC5E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAA;AAEjE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAC9C,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAEjE,yGAAyG;AACzG,MAAM,WAAW,WAAW;IAC3B,KAAK,EAAE,eAAe,CAAA;IACtB,MAAM,EAAE,cAAc,CAAA;CACtB;AAED,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,cAAc,CAAA;IACtB,IAAI,EAAE,YAAY,GAAG,QAAQ,CAAA;IAC7B,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,4DAA4D;IAC5D,QAAQ,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,CAAA;CAC3C;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACxC,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,eAAe,EACxB,IAAI,EAAE,WAAW,EACjB,QAAQ,EAAE,MAAM,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC,GACvD,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAyBnC"}
@@ -0,0 +1,34 @@
1
+ import { getCollectionContent } from "./cms-marker.js";
2
+ import { createCollectionOutput, createStaticOutput } from "./markdown-generator.js";
3
+ /**
4
+ * Resolve a page to the markdown that tells it: its collection entry when it has one, else its rendered
5
+ * HTML when static pages are wanted. `readPage` converts the rendered page, or gives `undefined` when
6
+ * there is none; it is called at most once and only when the page is needed.
7
+ */
8
+ export async function resolvePageMarkdown(pagePath, options, site, readPage) {
9
+ let page;
10
+ const renderedPage = () => (page ??= readPage());
11
+ const entry = await getCollectionContent(pagePath, site.pages, site.syntax);
12
+ if (entry) {
13
+ const entryTitle = entry.frontmatter.title?.value;
14
+ // An entry that renders more than its source says is told by its page, which is still the entry's,
15
+ // so it is written whether or not static pages are.
16
+ const rendered = entry.body === undefined || entryTitle === undefined ? await renderedPage() : undefined;
17
+ const body = entry.body ?? rendered?.body;
18
+ if (body === undefined)
19
+ return undefined;
20
+ return {
21
+ output: createCollectionOutput(entry.frontmatter, body, entry.file),
22
+ type: 'collection',
23
+ sourcePath: entry.file,
24
+ title: entryTitle ?? rendered?.metadata.title,
25
+ metadata: rendered?.metadata,
26
+ };
27
+ }
28
+ if (!options.includeStaticPages)
29
+ return undefined;
30
+ const rendered = await renderedPage();
31
+ if (!rendered)
32
+ return undefined;
33
+ return { output: createStaticOutput(rendered.metadata, rendered.body), type: 'static', title: rendered.metadata.title, metadata: rendered.metadata };
34
+ }
@@ -0,0 +1 @@
1
+ {"version":3,"file":"paths.d.ts","sourceRoot":"","sources":["../src/paths.ts"],"names":[],"mappings":"AAEA,iDAAiD;AACjD,eAAO,MAAM,iBAAiB,wBAAwB,CAAA;AAEtD,iCAAiC;AACjC,eAAO,MAAM,aAAa,cAAc,CAAA;AAExC;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMjD;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAMvD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMnD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKzE;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKrE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAExD;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAWzE"}
package/dist/paths.js ADDED
@@ -0,0 +1,79 @@
1
+ import path from 'node:path';
2
+ /** Well-known path for LLM discovery endpoint */
3
+ export const LLM_ENDPOINT_PATH = '/.well-known/llm.md';
4
+ /** Path for llms.txt endpoint */
5
+ export const LLMS_TXT_PATH = '/llms.txt';
6
+ /**
7
+ * Normalize a URL path by removing query strings, hashes, and trailing slashes
8
+ */
9
+ export function normalizePath(url) {
10
+ let pagePath = url.split('?')[0]?.split('#')[0] ?? '';
11
+ if (pagePath.length > 1 && pagePath.endsWith('/')) {
12
+ pagePath = pagePath.slice(0, -1);
13
+ }
14
+ return pagePath || '/';
15
+ }
16
+ /**
17
+ * Get the markdown URL for a given page path
18
+ */
19
+ export function getMarkdownUrl(pagePath) {
20
+ if (pagePath === '/') {
21
+ return '/index.md';
22
+ }
23
+ const cleanPath = pagePath.endsWith('/') ? pagePath.slice(0, -1) : pagePath;
24
+ return `${cleanPath}.md`;
25
+ }
26
+ /**
27
+ * Convert a .md URL back to a page path
28
+ */
29
+ export function mdUrlToPagePath(url) {
30
+ const match = url.match(/^\/(.*)\.md$/);
31
+ if (!match)
32
+ return url;
33
+ const pagePath = '/' + match[1];
34
+ return pagePath === '/index' ? '/' : pagePath;
35
+ }
36
+ /**
37
+ * Get the output path for a .md file in dist
38
+ */
39
+ export function getMdOutputPath(distDir, pagePath) {
40
+ if (pagePath === '/') {
41
+ return path.join(distDir, 'index.md');
42
+ }
43
+ return path.join(distDir, `${pagePath.slice(1)}.md`);
44
+ }
45
+ /**
46
+ * Get the HTML file path for a page in dist
47
+ */
48
+ export function getHtmlPath(distDir, pagePath) {
49
+ if (pagePath === '/') {
50
+ return path.join(distDir, 'index.html');
51
+ }
52
+ return path.join(distDir, pagePath.slice(1), 'index.html');
53
+ }
54
+ /**
55
+ * Get the output path for llm.md in dist
56
+ */
57
+ export function getLlmOutputPath(distDir) {
58
+ return path.join(distDir, '.well-known', 'llm.md');
59
+ }
60
+ /**
61
+ * Get the output path for llms.txt in dist
62
+ */
63
+ export function getLlmsTxtOutputPath(distDir) {
64
+ return path.join(distDir, 'llms.txt');
65
+ }
66
+ /**
67
+ * Inject markdown alternate link into HTML head
68
+ */
69
+ export function injectMarkdownLink(html, pagePath) {
70
+ const mdUrl = getMarkdownUrl(pagePath);
71
+ const linkTag = `<link rel="alternate" type="text/markdown" href="${mdUrl}">`;
72
+ if (html.includes('</head>')) {
73
+ return html.replace('</head>', `${linkTag}\n</head>`);
74
+ }
75
+ if (html.includes('<head>')) {
76
+ return html.replace('<head>', `<head>\n${linkTag}`);
77
+ }
78
+ return html;
79
+ }