jamdesk 1.1.32 → 1.1.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/vendored/app/[[...slug]]/page.tsx +3 -2
- package/vendored/app/api/markdown-export/[project]/[...slug]/route.ts +76 -0
- package/vendored/app/api/raw-content/[...slug]/route.ts +7 -6
- package/vendored/components/mdx/MDXComponents.tsx +7 -0
- package/vendored/components/mdx/Visibility.tsx +20 -0
- package/vendored/lib/content-negotiation.ts +29 -0
- package/vendored/lib/preprocess-mdx.ts +7 -0
- package/vendored/lib/process-mdx-with-exports.ts +2 -0
- package/vendored/lib/r2-content.ts +8 -0
- package/vendored/lib/remark-visibility.ts +86 -0
- package/vendored/lib/search.ts +4 -2
- package/vendored/lib/snippet-loader-isr.ts +5 -2
- package/vendored/lib/static-artifacts.ts +7 -2
- package/vendored/lib/visibility-filter.ts +162 -0
- package/vendored/scripts/build-search-index.cjs +4 -1
- package/vendored/scripts/visibility-filter.cjs +125 -0
- package/vendored/workspace-package-lock.json +7 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jamdesk",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.34",
|
|
4
4
|
"description": "CLI for Jamdesk — build, preview, and deploy documentation sites from MDX. Dev server with hot reload, 50+ components, OpenAPI support, AI search, and Mintlify migration",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"jamdesk",
|
|
@@ -41,6 +41,7 @@ import { PanelWrapper } from '@/components/mdx/PanelWrapper';
|
|
|
41
41
|
import { ViewWrapper } from '@/components/mdx/View';
|
|
42
42
|
import { getLatexRemarkPlugins, getLatexRehypePlugins } from '@/lib/latex-config';
|
|
43
43
|
import { getTypographyRemarkPlugins } from '@/lib/typography-config';
|
|
44
|
+
import { remarkVisibility } from '@/lib/remark-visibility';
|
|
44
45
|
import { recmaCompoundComponents } from '@/lib/recma-compound-components';
|
|
45
46
|
import { extractInlineComponents } from '@/lib/process-mdx-with-exports';
|
|
46
47
|
import { buildEndpointFromMdx } from '@/lib/build-endpoint-from-mdx';
|
|
@@ -698,7 +699,7 @@ export default async function DocPage({ params }: PageProps) {
|
|
|
698
699
|
// Keep expression props (e.g. cols={2}) compatible under next-mdx-remote v6.
|
|
699
700
|
...mdxSecurityOptions,
|
|
700
701
|
mdxOptions: {
|
|
701
|
-
remarkPlugins: [remarkGfm, ...getTypographyRemarkPlugins(config), ...getLatexRemarkPlugins(config)],
|
|
702
|
+
remarkPlugins: [remarkGfm, [remarkVisibility, { audience: 'humans' }], ...getTypographyRemarkPlugins(config), ...getLatexRemarkPlugins(config)],
|
|
702
703
|
rehypePlugins: [rehypeNoZoomToData, rehypeClassToClassName, rehypeCodeMeta, createShikiRehypePlugin(highlighter, config), rehypeRestoreDataTitle, ...getLatexRehypePlugins(config), rehypeSlug],
|
|
703
704
|
recmaPlugins: [recmaCompoundComponents],
|
|
704
705
|
},
|
|
@@ -724,7 +725,7 @@ export default async function DocPage({ params }: PageProps) {
|
|
|
724
725
|
// Keep expression props (e.g. cols={2}) compatible under next-mdx-remote v6.
|
|
725
726
|
...mdxSecurityOptions,
|
|
726
727
|
mdxOptions: {
|
|
727
|
-
remarkPlugins: [remarkGfm, ...getTypographyRemarkPlugins(config), ...getLatexRemarkPlugins(config)],
|
|
728
|
+
remarkPlugins: [remarkGfm, [remarkVisibility, { audience: 'humans' }], ...getTypographyRemarkPlugins(config), ...getLatexRemarkPlugins(config)],
|
|
728
729
|
rehypePlugins: [rehypeNoZoomToData, rehypeClassToClassName, rehypeCodeMeta, createShikiRehypePlugin(highlighter, config), rehypeRestoreDataTitle, ...getLatexRehypePlugins(config), rehypeSlug],
|
|
729
730
|
recmaPlugins: [recmaCompoundComponents],
|
|
730
731
|
},
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { NextResponse } from 'next/server';
|
|
2
|
+
import {
|
|
3
|
+
fetchMdxContent,
|
|
4
|
+
R2_NOT_FOUND_ERROR_NAMES,
|
|
5
|
+
R2_NOT_FOUND_MESSAGE_PREFIX,
|
|
6
|
+
} from '@/lib/r2-content';
|
|
7
|
+
import { filterVisibility } from '@/lib/visibility-filter';
|
|
8
|
+
import { VALID_SLUG_RE } from '@/lib/middleware-helpers';
|
|
9
|
+
import {
|
|
10
|
+
acceptsMarkdown,
|
|
11
|
+
MARKDOWN_CACHE_PUBLIC,
|
|
12
|
+
MARKDOWN_CACHE_PRIVATE,
|
|
13
|
+
} from '@/lib/content-negotiation';
|
|
14
|
+
|
|
15
|
+
export const runtime = 'nodejs';
|
|
16
|
+
export const dynamic = 'force-dynamic';
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Production `.md` export (and `Accept: text/markdown` canonical URLs).
|
|
20
|
+
* Fetches raw MDX from R2 via the existing in-memory-cached fetcher,
|
|
21
|
+
* applies the audience=agents filter, and returns as text/markdown with
|
|
22
|
+
* the same security headers the legacy /api/r2 path applied to .mdx.
|
|
23
|
+
*/
|
|
24
|
+
export async function GET(
|
|
25
|
+
request: Request,
|
|
26
|
+
{ params }: { params: Promise<{ project: string; slug: string[] }> }
|
|
27
|
+
) {
|
|
28
|
+
const { project, slug } = await params;
|
|
29
|
+
|
|
30
|
+
// Defense in depth: the proxy always resolves `project` from the domain
|
|
31
|
+
// resolver before routing here, but anything reachable via direct URL
|
|
32
|
+
// must be locked to the slug character set.
|
|
33
|
+
if (!VALID_SLUG_RE.test(project)) {
|
|
34
|
+
return new NextResponse('Bad request', { status: 400 });
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const pagePath = slug.join('/');
|
|
38
|
+
|
|
39
|
+
let raw: string;
|
|
40
|
+
try {
|
|
41
|
+
raw = await fetchMdxContent(project, pagePath);
|
|
42
|
+
} catch (err) {
|
|
43
|
+
const msg = err instanceof Error ? err.message : '';
|
|
44
|
+
const name = err instanceof Error ? err.name : '';
|
|
45
|
+
if (
|
|
46
|
+
msg.startsWith(R2_NOT_FOUND_MESSAGE_PREFIX) ||
|
|
47
|
+
R2_NOT_FOUND_ERROR_NAMES.has(name)
|
|
48
|
+
) {
|
|
49
|
+
return new NextResponse('Not found', { status: 404 });
|
|
50
|
+
}
|
|
51
|
+
return new NextResponse('Upstream unavailable', { status: 502 });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const filtered = filterVisibility(raw, 'agents');
|
|
55
|
+
|
|
56
|
+
// Two paths reach this handler via proxy rewrite:
|
|
57
|
+
// (a) `.md` URL: unique CDN cache key, safe to share.
|
|
58
|
+
// (b) canonical URL + `Accept: text/markdown`: shares a cache key with
|
|
59
|
+
// the HTML page. Cloudflare ignores `Vary: Accept` on public entries,
|
|
60
|
+
// so a cached markdown response would poison browser requests.
|
|
61
|
+
const cacheControl = acceptsMarkdown(request.headers.get('accept'))
|
|
62
|
+
? MARKDOWN_CACHE_PRIVATE
|
|
63
|
+
: MARKDOWN_CACHE_PUBLIC;
|
|
64
|
+
|
|
65
|
+
return new NextResponse(filtered, {
|
|
66
|
+
headers: {
|
|
67
|
+
'Content-Type': 'text/markdown; charset=utf-8',
|
|
68
|
+
'Content-Disposition': 'inline',
|
|
69
|
+
'Cache-Control': cacheControl,
|
|
70
|
+
'Vary': 'Accept',
|
|
71
|
+
'X-Robots-Tag': 'noindex, nofollow',
|
|
72
|
+
'X-Frame-Options': 'DENY',
|
|
73
|
+
'Content-Security-Policy': "default-src 'none'",
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
import { getContentLoader } from '@/lib/content-loader';
|
|
2
|
+
import { filterVisibility } from '@/lib/visibility-filter';
|
|
2
3
|
import { NextResponse } from 'next/server';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
6
|
+
* Dev `.md` route. Applies the visibility filter with audience='agents'
|
|
7
|
+
* — strips for="humans" blocks, unwraps for="agents" blocks. Code-block-safe.
|
|
8
|
+
* Production `.md` goes through /api/markdown-export/[project]/[...slug].
|
|
7
9
|
*/
|
|
8
10
|
export async function GET(
|
|
9
11
|
_request: Request,
|
|
10
12
|
{ params }: { params: Promise<{ slug: string[] }> }
|
|
11
13
|
) {
|
|
12
14
|
const { slug } = await params;
|
|
13
|
-
const pagePath = slug.join('/');
|
|
14
|
-
|
|
15
15
|
try {
|
|
16
16
|
const loader = getContentLoader();
|
|
17
|
-
const
|
|
18
|
-
|
|
17
|
+
const raw = await loader.getContent(slug.join('/'));
|
|
18
|
+
const filtered = filterVisibility(raw, 'agents');
|
|
19
|
+
return new NextResponse(filtered, {
|
|
19
20
|
headers: {
|
|
20
21
|
'Content-Type': 'text/markdown; charset=utf-8',
|
|
21
22
|
'Content-Disposition': 'inline',
|
|
@@ -31,6 +31,7 @@ import { Columns } from './Columns';
|
|
|
31
31
|
import { View, ViewProvider, ViewSelector, ViewWrapper } from './View';
|
|
32
32
|
import { YouTube } from './YouTube';
|
|
33
33
|
import { Video } from './Video';
|
|
34
|
+
import { Visibility } from './Visibility';
|
|
34
35
|
|
|
35
36
|
/**
|
|
36
37
|
* Extract language from a pre element for tab label
|
|
@@ -229,6 +230,12 @@ export const MDXComponents = {
|
|
|
229
230
|
YouTube,
|
|
230
231
|
// Video player for local video files
|
|
231
232
|
Video,
|
|
233
|
+
// Audience-conditional content. Filtered at MDX-compile time by
|
|
234
|
+
// lib/remark-visibility.ts for the HTML render path and by
|
|
235
|
+
// lib/visibility-filter.ts for text surfaces (.md export, llms-full.txt,
|
|
236
|
+
// search index). This React component is a fail-closed fallback for
|
|
237
|
+
// any <Visibility> that slips past both filters.
|
|
238
|
+
Visibility,
|
|
232
239
|
// Sized images from preprocess-mdx ( syntax).
|
|
233
240
|
// These are output as <SizedImage> JSX so they go through component mapping
|
|
234
241
|
// (raw <img> JSX in MDX bypasses the components provider).
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ReactNode } from 'react';
|
|
2
|
+
|
|
3
|
+
interface VisibilityProps {
|
|
4
|
+
for: 'humans' | 'agents';
|
|
5
|
+
children?: ReactNode;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* <Visibility for="humans|agents"> — audience-conditional content.
|
|
10
|
+
*
|
|
11
|
+
* Filtering is normally done at MDX-compile time by `lib/remark-visibility.ts`
|
|
12
|
+
* (HTML render, snippets) or at the text level by `lib/visibility-filter.ts`
|
|
13
|
+
* (.md export, llms-full.txt). This React component is a defensive passthrough
|
|
14
|
+
* so if the filter ever misses (dynamic import, future MDX form, writer bug),
|
|
15
|
+
* the component still fails-closed for agents and fails-visible for humans.
|
|
16
|
+
*/
|
|
17
|
+
export function Visibility({ for: audience, children }: VisibilityProps) {
|
|
18
|
+
if (audience === 'agents') return null;
|
|
19
|
+
return <>{children}</>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* True if the Accept header contains a `text/markdown` offer with a
|
|
3
|
+
* non-zero q-value (or no q-value). Scans offers individually so
|
|
4
|
+
* `text/markdown;q=0` ("explicitly not markdown") doesn't trigger a match.
|
|
5
|
+
*
|
|
6
|
+
* Shared between the proxy (for URL rewriting) and the markdown-export
|
|
7
|
+
* route handler (for cache policy decisions) so both layers agree on
|
|
8
|
+
* what counts as an agent-facing request.
|
|
9
|
+
*/
|
|
10
|
+
export function acceptsMarkdown(accept: string | null | undefined): boolean {
|
|
11
|
+
if (!accept) return false;
|
|
12
|
+
// Browsers never send text/markdown; short-circuit on the ~99% hot path
|
|
13
|
+
// without allocating a lowercased copy or splitting into offers.
|
|
14
|
+
if (accept.search(/text\/markdown/i) === -1) return false;
|
|
15
|
+
return accept
|
|
16
|
+
.toLowerCase()
|
|
17
|
+
.split(',')
|
|
18
|
+
.some(offer => {
|
|
19
|
+
const [type, ...params] = offer.trim().split(';').map(s => s.trim());
|
|
20
|
+
if (type !== 'text/markdown') return false;
|
|
21
|
+
const qParam = params.find(p => p.startsWith('q='));
|
|
22
|
+
if (!qParam) return true;
|
|
23
|
+
const q = Number.parseFloat(qParam.slice(2));
|
|
24
|
+
return Number.isFinite(q) && q > 0;
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export const MARKDOWN_CACHE_PUBLIC = 'public, max-age=3600, s-maxage=86400';
|
|
29
|
+
export const MARKDOWN_CACHE_PRIVATE = 'private, no-store';
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { ASSET_PREFIX, appendAssetVersion } from './docs-types';
|
|
10
10
|
import { checkForDeprecatedComponents } from './deprecated-components';
|
|
11
|
+
import { filterVisibility } from './visibility-filter';
|
|
11
12
|
|
|
12
13
|
/**
|
|
13
14
|
* JSX components that contain markdown content requiring special preprocessing.
|
|
@@ -35,6 +36,7 @@ const JSX_CONTENT_COMPONENTS = [
|
|
|
35
36
|
'View',
|
|
36
37
|
'Varning',
|
|
37
38
|
'Avertissement',
|
|
39
|
+
'Visibility',
|
|
38
40
|
] as const;
|
|
39
41
|
|
|
40
42
|
/**
|
|
@@ -935,6 +937,11 @@ export function preprocessMdx(content: string, options?: { assetVersion?: string
|
|
|
935
937
|
let processed = content;
|
|
936
938
|
const assetVersion = options?.assetVersion;
|
|
937
939
|
|
|
940
|
+
// Audience filter (code-block-safe). The remark plugin in page.tsx is the
|
|
941
|
+
// primary line of defense for HTML render; this backs up snippet files
|
|
942
|
+
// that flow through preprocessMdx independently.
|
|
943
|
+
processed = filterVisibility(processed, 'humans');
|
|
944
|
+
|
|
938
945
|
// Strip snippet imports
|
|
939
946
|
processed = stripSnippetImports(processed);
|
|
940
947
|
|
|
@@ -12,6 +12,7 @@ import remarkParse from 'remark-parse';
|
|
|
12
12
|
import remarkMdx from 'remark-mdx';
|
|
13
13
|
import { VFile } from 'vfile';
|
|
14
14
|
import { remarkExtractExports } from './remark-extract-exports';
|
|
15
|
+
import { remarkVisibility } from './remark-visibility';
|
|
15
16
|
import { compileInlineComponents } from './mdx-inline-components';
|
|
16
17
|
import { remarkExtractParamFields } from './remark-extract-param-fields';
|
|
17
18
|
import type { ExtractedParam } from './remark-extract-param-fields';
|
|
@@ -30,6 +31,7 @@ interface ProcessResult {
|
|
|
30
31
|
const mdxProcessor = unified()
|
|
31
32
|
.use(remarkParse)
|
|
32
33
|
.use(remarkMdx)
|
|
34
|
+
.use(remarkVisibility, { audience: 'humans' })
|
|
33
35
|
.use(remarkExtractExports)
|
|
34
36
|
.use(remarkExtractParamFields);
|
|
35
37
|
|
|
@@ -9,6 +9,12 @@ import type { DocsConfig } from './docs-types';
|
|
|
9
9
|
|
|
10
10
|
export type { DocsConfig };
|
|
11
11
|
|
|
12
|
+
export const R2_NOT_FOUND_ERROR_NAMES: ReadonlySet<string> = new Set([
|
|
13
|
+
'NoSuchKey',
|
|
14
|
+
'AccessDenied',
|
|
15
|
+
]);
|
|
16
|
+
export const R2_NOT_FOUND_MESSAGE_PREFIX = 'Page not found';
|
|
17
|
+
|
|
12
18
|
export function evictOldest<K, V>(cache: Map<K, V>, maxSize: number): void {
|
|
13
19
|
if (cache.size <= maxSize) return;
|
|
14
20
|
const toDelete = cache.size - maxSize;
|
|
@@ -23,6 +29,8 @@ export function evictOldest<K, V>(cache: Map<K, V>, maxSize: number): void {
|
|
|
23
29
|
export function clearConfigCache(_projectSlug?: string): void {}
|
|
24
30
|
export function getConfigCacheSize(): number { return 0; }
|
|
25
31
|
|
|
32
|
+
export function isR2NotFound(_err: unknown): boolean { return false; }
|
|
33
|
+
|
|
26
34
|
let fetchDocsConfigWarned = false;
|
|
27
35
|
export async function fetchDocsConfig(_projectSlug: string): Promise<DocsConfig | null> {
|
|
28
36
|
if (!fetchDocsConfigWarned) {
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* remark-visibility
|
|
3
|
+
*
|
|
4
|
+
* MDX AST plugin that strips or unwraps <Visibility for="humans|agents">
|
|
5
|
+
* elements based on the consuming audience. Run this plugin BEFORE
|
|
6
|
+
* remarkExtractExports so exports inside filtered-out blocks are
|
|
7
|
+
* dropped along with the block.
|
|
8
|
+
*
|
|
9
|
+
* Unlike the text-level filter in lib/visibility-filter.ts, this plugin
|
|
10
|
+
* is safe for:
|
|
11
|
+
* - code blocks (mdast `code` nodes aren't traversed as JSX)
|
|
12
|
+
* - nested <Visibility> elements
|
|
13
|
+
* - JSX expressions
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { Root, RootContent } from 'mdast';
|
|
17
|
+
import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx';
|
|
18
|
+
|
|
19
|
+
type JsxEl = MdxJsxFlowElement | MdxJsxTextElement;
|
|
20
|
+
|
|
21
|
+
export interface RemarkVisibilityOptions {
|
|
22
|
+
audience: 'humans' | 'agents';
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function isVisibility(node: unknown): node is JsxEl {
|
|
26
|
+
if (!node || typeof node !== 'object') return false;
|
|
27
|
+
const n = node as { type?: string; name?: string };
|
|
28
|
+
return (
|
|
29
|
+
(n.type === 'mdxJsxFlowElement' || n.type === 'mdxJsxTextElement') &&
|
|
30
|
+
n.name === 'Visibility'
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function getForAttr(node: JsxEl): string | null {
|
|
35
|
+
for (const attr of node.attributes ?? []) {
|
|
36
|
+
if (attr.type === 'mdxJsxAttribute' && attr.name === 'for') {
|
|
37
|
+
if (typeof attr.value === 'string') return attr.value.toLowerCase();
|
|
38
|
+
// Expression-valued attribute — treat as unknown, leave alone.
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function remarkVisibility(options: RemarkVisibilityOptions) {
|
|
46
|
+
const { audience } = options;
|
|
47
|
+
return function transform(tree: Root): void {
|
|
48
|
+
visitInPlace(tree, audience);
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function visitInPlace(
|
|
53
|
+
parent: { children?: RootContent[] },
|
|
54
|
+
audience: 'humans' | 'agents',
|
|
55
|
+
): void {
|
|
56
|
+
if (!Array.isArray(parent.children)) return;
|
|
57
|
+
|
|
58
|
+
const next: RootContent[] = [];
|
|
59
|
+
for (const child of parent.children) {
|
|
60
|
+
if (isVisibility(child)) {
|
|
61
|
+
const target = getForAttr(child);
|
|
62
|
+
if (target === null) {
|
|
63
|
+
visitInPlace(child as unknown as { children?: RootContent[] }, audience);
|
|
64
|
+
next.push(child);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (target !== 'humans' && target !== 'agents') {
|
|
68
|
+
visitInPlace(child as unknown as { children?: RootContent[] }, audience);
|
|
69
|
+
next.push(child);
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (target === audience) {
|
|
73
|
+
// Unwrap: recurse first so nested <Visibility> get filtered, then splice in children.
|
|
74
|
+
visitInPlace(child as unknown as { children?: RootContent[] }, audience);
|
|
75
|
+
next.push(...((child.children ?? []) as RootContent[]));
|
|
76
|
+
}
|
|
77
|
+
// Else: drop entirely.
|
|
78
|
+
continue;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
visitInPlace(child as unknown as { children?: RootContent[] }, audience);
|
|
82
|
+
next.push(child);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
parent.children = next;
|
|
86
|
+
}
|
package/vendored/lib/search.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { parseFrontmatterLenient } from './frontmatter-utils';
|
|
4
|
+
import { filterVisibility } from './visibility-filter';
|
|
4
5
|
|
|
5
6
|
export interface SearchResult {
|
|
6
7
|
id: string;
|
|
@@ -79,8 +80,9 @@ export function buildSearchIndex(): SearchResult[] {
|
|
|
79
80
|
const fileContents = fs.readFileSync(filePath, 'utf8');
|
|
80
81
|
const { data, content } = parseFrontmatterLenient(fileContents);
|
|
81
82
|
|
|
82
|
-
//
|
|
83
|
-
const
|
|
83
|
+
// Filter for="agents" content out of the search index.
|
|
84
|
+
const visibleContent = filterVisibility(content, 'humans');
|
|
85
|
+
const sections = extractSections(visibleContent);
|
|
84
86
|
|
|
85
87
|
sections.forEach((section, idx) => {
|
|
86
88
|
const cleanContent = stripMarkdown(section.content);
|
|
@@ -14,6 +14,7 @@ import * as jsxRuntime from 'react/jsx-runtime';
|
|
|
14
14
|
import { transform } from '@babel/standalone';
|
|
15
15
|
import { fetchSnippet } from './r2-content';
|
|
16
16
|
import { extractSnippetImports } from './preprocess-mdx';
|
|
17
|
+
import { filterVisibility } from './visibility-filter';
|
|
17
18
|
|
|
18
19
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
19
20
|
type AnyComponent = React.ComponentType<any>;
|
|
@@ -240,8 +241,10 @@ async function compileSnippet(
|
|
|
240
241
|
return cached.result;
|
|
241
242
|
}
|
|
242
243
|
|
|
243
|
-
// Fetch from R2
|
|
244
|
-
|
|
244
|
+
// Fetch from R2 and strip <Visibility for="agents"> blocks before compiling.
|
|
245
|
+
// Filter before caching so cached components are already audience-filtered.
|
|
246
|
+
const rawSource = await fetchSnippet(projectSlug, snippetPath);
|
|
247
|
+
const source = filterVisibility(rawSource, 'humans');
|
|
245
248
|
|
|
246
249
|
// Check for 'use client' directive
|
|
247
250
|
const trimmed = source.trimStart();
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import type { NavigationConfig } from './docs-types.js';
|
|
9
9
|
import { RECURSE_KEYS } from './enhance-navigation.js';
|
|
10
|
+
import { filterVisibility } from './visibility-filter.js';
|
|
10
11
|
|
|
11
12
|
/**
|
|
12
13
|
* Page metadata for artifact generation.
|
|
@@ -313,7 +314,7 @@ export function generateLlmsFullTxt(options: LlmsFullTxtOptions): string {
|
|
|
313
314
|
parts.push(`*${page.frontmatter.description}*\n\n`);
|
|
314
315
|
}
|
|
315
316
|
|
|
316
|
-
const content = page.content.trim();
|
|
317
|
+
const content = filterVisibility(page.content, 'agents').trim();
|
|
317
318
|
if (content) {
|
|
318
319
|
parts.push(`${content}\n\n`);
|
|
319
320
|
}
|
|
@@ -676,7 +677,11 @@ export function generateSearchData(pages: SearchPageInfo[]): string {
|
|
|
676
677
|
const pathWithoutExt = page.path.replace(/\.mdx?$/, '');
|
|
677
678
|
const slug = pathWithoutExt.replace(/\\/g, '/');
|
|
678
679
|
const pageType = inferPageType(slug);
|
|
679
|
-
|
|
680
|
+
// Filter for="agents" content out of the search index — the site
|
|
681
|
+
// search is a human-facing surface, so agent-only content must not
|
|
682
|
+
// leak into autocomplete.
|
|
683
|
+
const visibleContent = filterVisibility(page.content, 'humans');
|
|
684
|
+
const sections = extractSections(visibleContent);
|
|
680
685
|
|
|
681
686
|
sections.forEach((section, idx) => {
|
|
682
687
|
const cleanContent = stripMarkdown(section.content);
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Visibility text-level filter — used by the prod `.md` export route, the
|
|
3
|
+
* search-index generator, and the CJS build scripts, which can't run the
|
|
4
|
+
* MDX AST pipeline.
|
|
5
|
+
*
|
|
6
|
+
* For HTML render and snippets, the remark plugin at `lib/remark-visibility.ts`
|
|
7
|
+
* is used instead (AST-safe; handles nested tags and JSX expressions).
|
|
8
|
+
*
|
|
9
|
+
* This filter preserves:
|
|
10
|
+
* - Fenced code blocks (``` and ~~~), with or without trailing whitespace
|
|
11
|
+
* after the closer
|
|
12
|
+
* - CommonMark indented code blocks (4+ spaces or tab, preceded by a
|
|
13
|
+
* blank line or start-of-string)
|
|
14
|
+
* - Inline code (`...`)
|
|
15
|
+
*
|
|
16
|
+
* KNOWN LIMITATION: this filter is not nesting-safe. A non-greedy regex
|
|
17
|
+
* always consumes to the first </Visibility>. The fixpoint loop handles
|
|
18
|
+
* sibling blocks produced by unwrap, but nested <Visibility> in the text
|
|
19
|
+
* surface (.md export, llms-full.txt) produces mangled output. Customer
|
|
20
|
+
* docs discourage nesting. For the HTML render surface the remark plugin
|
|
21
|
+
* handles nesting correctly.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export type VisibilityAudience = 'humans' | 'agents';
|
|
25
|
+
|
|
26
|
+
// Attribute region tolerates `>` inside quoted values (writer-legal MDX
|
|
27
|
+
// like `<Visibility title="a>b">`). We alternate quoted strings vs
|
|
28
|
+
// non-special chars so `>` in a quoted value can't prematurely close the
|
|
29
|
+
// tag. Pattern: `"..."`, `'...'`, or a char that's not `>`/`"`/`'`.
|
|
30
|
+
const VISIBILITY_TAG =
|
|
31
|
+
/<Visibility\s+((?:"[^"]*"|'[^']*'|[^>"'])*?)(?:\/>|>([\s\S]*?)<\/Visibility>)/g;
|
|
32
|
+
// Anchor `for` on whitespace / start-of-attrs so attributes like
|
|
33
|
+
// `data-for="x"` or `my-for="x"` can't collide.
|
|
34
|
+
const FOR_ATTR = /(?:^|\s)for\s*=\s*["']([^"']+)["']/;
|
|
35
|
+
|
|
36
|
+
// Closing fence tolerates trailing whitespace (CommonMark §4.5).
|
|
37
|
+
const FENCED_BLOCK = /(^|\n)(```|~~~)[^\n]*\n[\s\S]*?\n\2[ \t]*(?=\n|$)/g;
|
|
38
|
+
const INLINE_CODE = /`[^`\n]+`/g;
|
|
39
|
+
const INDENTED_LINE = /^(?: {4,}|\t)/;
|
|
40
|
+
// Cheap pre-check: indented code blocks require a \n followed by 4+ spaces
|
|
41
|
+
// or a tab. If neither appears, skip the line-split + per-line scan below.
|
|
42
|
+
const INDENT_PRECHECK = /\n(?: {4}|\t)/;
|
|
43
|
+
|
|
44
|
+
// Fixpoint-loop cap. Depth of re-matches in any real document is <<10;
|
|
45
|
+
// this guards against pathological / adversarial input.
|
|
46
|
+
const MAX_PASSES = 16;
|
|
47
|
+
|
|
48
|
+
// CommonMark indented code blocks (4+ spaces or a tab, preceded by a blank
|
|
49
|
+
// line or start-of-string). Returns half-open offset ranges. Trailing blank
|
|
50
|
+
// lines inside a block are excluded from the preserved range.
|
|
51
|
+
function findIndentedCodeRanges(content: string): Array<[number, number]> {
|
|
52
|
+
if (!INDENT_PRECHECK.test(content) && !INDENTED_LINE.test(content)) {
|
|
53
|
+
return [];
|
|
54
|
+
}
|
|
55
|
+
const ranges: Array<[number, number]> = [];
|
|
56
|
+
const lines = content.split('\n');
|
|
57
|
+
const offsets: number[] = new Array(lines.length + 1);
|
|
58
|
+
offsets[0] = 0;
|
|
59
|
+
for (let i = 0; i < lines.length; i++) {
|
|
60
|
+
offsets[i + 1] = offsets[i] + lines[i].length + 1;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
let blockStart = -1;
|
|
64
|
+
let trailingBlanks = 0;
|
|
65
|
+
let prevBlank = true; // start-of-string qualifies as "blank" for trigger
|
|
66
|
+
for (let i = 0; i < lines.length; i++) {
|
|
67
|
+
const line = lines[i];
|
|
68
|
+
const isBlank = line.trim() === '';
|
|
69
|
+
const isIndented = INDENTED_LINE.test(line);
|
|
70
|
+
if (blockStart < 0) {
|
|
71
|
+
if (prevBlank && isIndented && !isBlank) {
|
|
72
|
+
blockStart = i;
|
|
73
|
+
trailingBlanks = 0;
|
|
74
|
+
}
|
|
75
|
+
} else if (isIndented && !isBlank) {
|
|
76
|
+
trailingBlanks = 0;
|
|
77
|
+
} else if (isBlank) {
|
|
78
|
+
trailingBlanks++;
|
|
79
|
+
} else {
|
|
80
|
+
ranges.push([offsets[blockStart], offsets[i - trailingBlanks]]);
|
|
81
|
+
blockStart = -1;
|
|
82
|
+
trailingBlanks = 0;
|
|
83
|
+
}
|
|
84
|
+
prevBlank = isBlank;
|
|
85
|
+
}
|
|
86
|
+
if (blockStart >= 0) {
|
|
87
|
+
const end = lines.length - trailingBlanks;
|
|
88
|
+
ranges.push([offsets[blockStart], offsets[end]]);
|
|
89
|
+
}
|
|
90
|
+
return ranges;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function transformInlineAware(content: string, fn: (text: string) => string): string {
|
|
94
|
+
const out: string[] = [];
|
|
95
|
+
let last = 0;
|
|
96
|
+
for (const m of content.matchAll(INLINE_CODE)) {
|
|
97
|
+
const idx = m.index ?? 0;
|
|
98
|
+
out.push(fn(content.slice(last, idx)));
|
|
99
|
+
out.push(m[0]);
|
|
100
|
+
last = idx + m[0].length;
|
|
101
|
+
}
|
|
102
|
+
out.push(fn(content.slice(last)));
|
|
103
|
+
return out.join('');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function transformOutsideIndentedCode(content: string, fn: (text: string) => string): string {
|
|
107
|
+
const ranges = findIndentedCodeRanges(content);
|
|
108
|
+
if (ranges.length === 0) return transformInlineAware(content, fn);
|
|
109
|
+
const parts: string[] = [];
|
|
110
|
+
let last = 0;
|
|
111
|
+
for (const [start, end] of ranges) {
|
|
112
|
+
parts.push(transformInlineAware(content.slice(last, start), fn));
|
|
113
|
+
parts.push(content.slice(start, end));
|
|
114
|
+
last = end;
|
|
115
|
+
}
|
|
116
|
+
parts.push(transformInlineAware(content.slice(last), fn));
|
|
117
|
+
return parts.join('');
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Apply a transform only to regions OUTSIDE fenced, indented, and inline code.
|
|
121
|
+
function transformOutsideCodeBlocks(
|
|
122
|
+
content: string,
|
|
123
|
+
fn: (text: string) => string,
|
|
124
|
+
): string {
|
|
125
|
+
const parts: string[] = [];
|
|
126
|
+
let last = 0;
|
|
127
|
+
for (const m of content.matchAll(FENCED_BLOCK)) {
|
|
128
|
+
const idx = m.index ?? 0;
|
|
129
|
+
parts.push(transformOutsideIndentedCode(content.slice(last, idx), fn));
|
|
130
|
+
parts.push(m[0]);
|
|
131
|
+
last = idx + m[0].length;
|
|
132
|
+
}
|
|
133
|
+
parts.push(transformOutsideIndentedCode(content.slice(last), fn));
|
|
134
|
+
return parts.join('');
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function filterVisibilityRaw(content: string, audience: VisibilityAudience): string {
|
|
138
|
+
return content.replace(VISIBILITY_TAG, (match, attrs: string, inner: string | undefined) => {
|
|
139
|
+
const forMatch = attrs.match(FOR_ATTR);
|
|
140
|
+
if (!forMatch) return match;
|
|
141
|
+
const target = forMatch[1].toLowerCase();
|
|
142
|
+
if (target !== 'humans' && target !== 'agents') return match;
|
|
143
|
+
if (target === audience) return inner ?? '';
|
|
144
|
+
return '';
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Filter <Visibility> blocks from MDX text. Safe for fenced, indented, and
|
|
150
|
+
* inline code. Iterates to a fixpoint so unwrap passes surface newly-
|
|
151
|
+
* adjacent tags.
|
|
152
|
+
*/
|
|
153
|
+
export function filterVisibility(content: string, audience: VisibilityAudience): string {
|
|
154
|
+
let prev: string;
|
|
155
|
+
let out = content;
|
|
156
|
+
for (let i = 0; i < MAX_PASSES; i++) {
|
|
157
|
+
prev = out;
|
|
158
|
+
out = transformOutsideCodeBlocks(prev, t => filterVisibilityRaw(t, audience));
|
|
159
|
+
if (out === prev) break;
|
|
160
|
+
}
|
|
161
|
+
return out;
|
|
162
|
+
}
|
|
@@ -23,6 +23,7 @@ const matter = require('gray-matter');
|
|
|
23
23
|
const os = require('os');
|
|
24
24
|
const { create, insertMultiple } = require('@orama/orama');
|
|
25
25
|
const { persist } = require('@orama/plugin-data-persistence');
|
|
26
|
+
const { filterVisibility } = require('./visibility-filter.cjs');
|
|
26
27
|
|
|
27
28
|
// Concurrency control for parallel file processing
|
|
28
29
|
const CONCURRENCY = parseInt(process.env.SEARCH_INDEX_CONCURRENCY || '') || os.cpus().length * 2;
|
|
@@ -256,7 +257,9 @@ async function buildSearchIndex() {
|
|
|
256
257
|
try {
|
|
257
258
|
const fileContents = await fsPromises.readFile(filePath, 'utf8');
|
|
258
259
|
const { data, content } = parseFrontmatterLenient(fileContents);
|
|
259
|
-
|
|
260
|
+
// Filter for="agents" content out of the search index.
|
|
261
|
+
const visibleContent = filterVisibility(content, 'humans');
|
|
262
|
+
const sections = extractSections(visibleContent);
|
|
260
263
|
|
|
261
264
|
const docs = [];
|
|
262
265
|
const normalizedSlug = slug.replace(/\\/g, '/');
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CJS mirror of lib/visibility-filter.ts.
|
|
3
|
+
*
|
|
4
|
+
* Used by CJS build scripts (generate-llms-full.cjs, build-search-index.cjs)
|
|
5
|
+
* that can't import the ESM TypeScript source. Behavior must stay
|
|
6
|
+
* byte-for-byte identical to the TS filter so that runtime and build-time
|
|
7
|
+
* outputs match (see `__tests__/lib/visibility-filter.test.ts` for the
|
|
8
|
+
* behaviors we pin).
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
'use strict';
|
|
12
|
+
|
|
13
|
+
const VIS_TAG =
|
|
14
|
+
/<Visibility\s+((?:"[^"]*"|'[^']*'|[^>"'])*?)(?:\/>|>([\s\S]*?)<\/Visibility>)/g;
|
|
15
|
+
const VIS_FOR = /(?:^|\s)for\s*=\s*["']([^"']+)["']/;
|
|
16
|
+
const VIS_FENCED = /(^|\n)(```|~~~)[^\n]*\n[\s\S]*?\n\2[ \t]*(?=\n|$)/g;
|
|
17
|
+
const VIS_INLINE = /`[^`\n]+`/g;
|
|
18
|
+
const VIS_INDENTED_LINE = /^(?: {4,}|\t)/;
|
|
19
|
+
const VIS_INDENT_PRECHECK = /\n(?: {4}|\t)/;
|
|
20
|
+
const VIS_MAX_PASSES = 16;
|
|
21
|
+
|
|
22
|
+
function findIndentedCodeRanges(content) {
|
|
23
|
+
if (!VIS_INDENT_PRECHECK.test(content) && !VIS_INDENTED_LINE.test(content)) {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
const ranges = [];
|
|
27
|
+
const lines = content.split('\n');
|
|
28
|
+
const offsets = new Array(lines.length + 1);
|
|
29
|
+
offsets[0] = 0;
|
|
30
|
+
for (let i = 0; i < lines.length; i++) {
|
|
31
|
+
offsets[i + 1] = offsets[i] + lines[i].length + 1;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
let blockStart = -1;
|
|
35
|
+
let trailingBlanks = 0;
|
|
36
|
+
let prevBlank = true;
|
|
37
|
+
for (let i = 0; i < lines.length; i++) {
|
|
38
|
+
const line = lines[i];
|
|
39
|
+
const isBlank = line.trim() === '';
|
|
40
|
+
const isIndented = VIS_INDENTED_LINE.test(line);
|
|
41
|
+
if (blockStart < 0) {
|
|
42
|
+
if (prevBlank && isIndented && !isBlank) {
|
|
43
|
+
blockStart = i;
|
|
44
|
+
trailingBlanks = 0;
|
|
45
|
+
}
|
|
46
|
+
} else if (isIndented && !isBlank) {
|
|
47
|
+
trailingBlanks = 0;
|
|
48
|
+
} else if (isBlank) {
|
|
49
|
+
trailingBlanks++;
|
|
50
|
+
} else {
|
|
51
|
+
ranges.push([offsets[blockStart], offsets[i - trailingBlanks]]);
|
|
52
|
+
blockStart = -1;
|
|
53
|
+
trailingBlanks = 0;
|
|
54
|
+
}
|
|
55
|
+
prevBlank = isBlank;
|
|
56
|
+
}
|
|
57
|
+
if (blockStart >= 0) {
|
|
58
|
+
const end = lines.length - trailingBlanks;
|
|
59
|
+
ranges.push([offsets[blockStart], offsets[end]]);
|
|
60
|
+
}
|
|
61
|
+
return ranges;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function filterVisibilityRaw(content, audience) {
|
|
65
|
+
return content.replace(VIS_TAG, (match, attrs, inner) => {
|
|
66
|
+
const m = attrs.match(VIS_FOR);
|
|
67
|
+
if (!m) return match;
|
|
68
|
+
const t = m[1].toLowerCase();
|
|
69
|
+
if (t !== 'humans' && t !== 'agents') return match;
|
|
70
|
+
if (t === audience) return inner == null ? '' : inner;
|
|
71
|
+
return '';
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function transformInlineAware(content, fn) {
|
|
76
|
+
const out = [];
|
|
77
|
+
let last = 0;
|
|
78
|
+
for (const m of content.matchAll(VIS_INLINE)) {
|
|
79
|
+
const idx = m.index ?? 0;
|
|
80
|
+
out.push(fn(content.slice(last, idx)));
|
|
81
|
+
out.push(m[0]);
|
|
82
|
+
last = idx + m[0].length;
|
|
83
|
+
}
|
|
84
|
+
out.push(fn(content.slice(last)));
|
|
85
|
+
return out.join('');
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function transformOutsideIndentedCode(content, fn) {
|
|
89
|
+
const ranges = findIndentedCodeRanges(content);
|
|
90
|
+
if (ranges.length === 0) return transformInlineAware(content, fn);
|
|
91
|
+
const parts = [];
|
|
92
|
+
let last = 0;
|
|
93
|
+
for (const [start, end] of ranges) {
|
|
94
|
+
parts.push(transformInlineAware(content.slice(last, start), fn));
|
|
95
|
+
parts.push(content.slice(start, end));
|
|
96
|
+
last = end;
|
|
97
|
+
}
|
|
98
|
+
parts.push(transformInlineAware(content.slice(last), fn));
|
|
99
|
+
return parts.join('');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function filterVisibilityOnce(content, audience) {
|
|
103
|
+
const parts = [];
|
|
104
|
+
let last = 0;
|
|
105
|
+
for (const m of content.matchAll(VIS_FENCED)) {
|
|
106
|
+
const idx = m.index ?? 0;
|
|
107
|
+
parts.push(transformOutsideIndentedCode(content.slice(last, idx), t => filterVisibilityRaw(t, audience)));
|
|
108
|
+
parts.push(m[0]);
|
|
109
|
+
last = idx + m[0].length;
|
|
110
|
+
}
|
|
111
|
+
parts.push(transformOutsideIndentedCode(content.slice(last), t => filterVisibilityRaw(t, audience)));
|
|
112
|
+
return parts.join('');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function filterVisibility(content, audience) {
|
|
116
|
+
let out = content;
|
|
117
|
+
for (let i = 0; i < VIS_MAX_PASSES; i++) {
|
|
118
|
+
const prev = out;
|
|
119
|
+
out = filterVisibilityOnce(prev, audience);
|
|
120
|
+
if (out === prev) break;
|
|
121
|
+
}
|
|
122
|
+
return out;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
module.exports = { filterVisibility };
|
|
@@ -2980,19 +2980,19 @@
|
|
|
2980
2980
|
"license": "MIT"
|
|
2981
2981
|
},
|
|
2982
2982
|
"node_modules/electron-to-chromium": {
|
|
2983
|
-
"version": "1.5.
|
|
2984
|
-
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.
|
|
2985
|
-
"integrity": "sha512-
|
|
2983
|
+
"version": "1.5.344",
|
|
2984
|
+
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.344.tgz",
|
|
2985
|
+
"integrity": "sha512-4MxfbmNDm+KPh066EZy+eUnkcDPcZ35wNmOWzFuh/ijvHsve6kbLTLURy88uCNK5FbpN+yk2nQY6BYh1GEt+wg==",
|
|
2986
2986
|
"license": "ISC"
|
|
2987
2987
|
},
|
|
2988
2988
|
"node_modules/enhanced-resolve": {
|
|
2989
|
-
"version": "5.
|
|
2990
|
-
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.
|
|
2991
|
-
"integrity": "sha512-
|
|
2989
|
+
"version": "5.21.0",
|
|
2990
|
+
"resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.21.0.tgz",
|
|
2991
|
+
"integrity": "sha512-otxSQPw4lkOZWkHpB3zaEQs6gWYEsmX4xQF68ElXC/TWvGxGMSGOvoNbaLXm6/cS/fSfHtsEdw90y20PCd+sCA==",
|
|
2992
2992
|
"license": "MIT",
|
|
2993
2993
|
"dependencies": {
|
|
2994
2994
|
"graceful-fs": "^4.2.4",
|
|
2995
|
-
"tapable": "^2.3.
|
|
2995
|
+
"tapable": "^2.3.3"
|
|
2996
2996
|
},
|
|
2997
2997
|
"engines": {
|
|
2998
2998
|
"node": ">=10.13.0"
|