@ctcstack/ctcdocs-core 0.16.2 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -3,6 +3,8 @@
3
3
  The configuration layer shared by the [CTCDocs](https://github.com/ctcstack/ctcdocs)
4
4
  packages: the schema of a project's `site.config.json`, the fixed project
5
5
  layout, and the allowlist of paths the synchronization pipeline may write.
6
+ Its `published-markdown` entry serializes a page's Markdown version, which
7
+ the site serves and the sync measures.
6
8
 
7
9
  A project does not install this package directly — `@ctcstack/ctcdocs` and
8
10
  `@ctcstack/ctcdocs-sync` depend on it. It is documented because what it
@@ -19,6 +19,10 @@ export interface CorpusDocument {
19
19
  * on its new chain to be confirmed.
20
20
  */
21
21
  readonly publishedReaders?: '*' | readonly string[];
22
+ /** The permanent short ID (ADR-022), the title and Drive's modified time. */
23
+ readonly shortId?: string;
24
+ readonly title?: string;
25
+ readonly modified?: string;
22
26
  }
23
27
  export interface CorpusStructure {
24
28
  readonly rootFolderId: string | null;
@@ -57,11 +57,21 @@ export function parseCorpusStructure(manifest) {
57
57
  const documents = new Map();
58
58
  for (const [id, raw] of entries(source.documents, 'documents')) {
59
59
  const path = `documents.${id}`;
60
- const published = raw.publishedReaders;
60
+ const record = raw;
61
+ const published = record.publishedReaders;
62
+ const optional = (key) => typeof record[key] === 'string' && record[key].length > 0
63
+ ? record[key]
64
+ : undefined;
65
+ const shortId = optional('shortId');
66
+ const title = optional('displayTitle');
67
+ const modified = optional('googleModifiedTime');
61
68
  documents.set(id, {
62
69
  id,
63
70
  parentId: field(raw, 'googleParentId', path, true),
64
71
  slug: field(raw, 'stableSlug', path) ?? '',
72
+ ...(shortId ? { shortId } : {}),
73
+ ...(title ? { title } : {}),
74
+ ...(modified ? { modified } : {}),
65
75
  ...(published === '*' ||
66
76
  (Array.isArray(published) &&
67
77
  published.every((group) => typeof group === 'string'))
package/dist/index.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- export { ACCESS_CLASSES_ROUTE, GENERATED_DIRECTORY_ALLOWLIST, GENERATED_FILE_ALLOWLIST, PLATFORM_ROUTE_HREFS, PLATFORM_ROUTES, PLATFORM_WORKERS, permanentLinkPath, PROJECT_LAYOUT, RESERVED_SLUGS, SHORT_ID_PATTERN, } from './project-layout.js';
1
+ export { ACCESS_CLASSES_ROUTE, GENERATED_DIRECTORY_ALLOWLIST, GENERATED_FILE_ALLOWLIST, markdownProjectionPath, PLATFORM_ROUTE_HREFS, PLATFORM_ROUTES, PLATFORM_WORKERS, permanentLinkPath, PROJECT_LAYOUT, RESERVED_SLUGS, resolvePermanentLink, SHORT_ID_PATTERN, } from './project-layout.js';
2
2
  export { assertGeneratedPathAllowed, isGeneratedPathAllowed, normalizeRepositoryPath, } from './generated-paths.js';
3
3
  export { generatedMarkdownHeader, generatedSourceHeader, } from './ownership-markers.js';
4
4
  export { findProjectRoot, ProjectRootError } from './project-root.js';
@@ -10,5 +10,5 @@ export { ADMINS_CLASS, chainReaders, chainRules, classIdentifiers, computeAccess
10
10
  export type { AccessClass, AccessModel, ChainReaders, PublishedReaders, Readers, } from './access-classes.js';
11
11
  export { accessFindings } from './access-findings.js';
12
12
  export type { AccessFinding } from './access-findings.js';
13
- export { loadSiteConfiguration, parseSiteConfiguration, SiteConfigurationError, } from './site-configuration.js';
14
- export type { AddressPolicy, BrandConfiguration, DeploymentConfiguration, DeploymentEnvironmentConfiguration, DeploymentEnvironmentConfigurations, DeploymentVisibility, HomeConfiguration, NavigationConfiguration, SignInConfiguration, SiteConfiguration, SyncConfigurationDefaults, } from './site-configuration.js';
13
+ export { fetchCharacterLimit, loadSiteConfiguration, MCP_DEFAULTS, parseSiteConfiguration, SiteConfigurationError, } from './site-configuration.js';
14
+ export type { AddressPolicy, BrandConfiguration, DeploymentConfiguration, DeploymentEnvironmentConfiguration, DeploymentEnvironmentConfigurations, DeploymentVisibility, HomeConfiguration, McpConfiguration, McpSearchConfiguration, NavigationConfiguration, SignInConfiguration, SiteConfiguration, SyncConfigurationDefaults, } from './site-configuration.js';
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- export { ACCESS_CLASSES_ROUTE, GENERATED_DIRECTORY_ALLOWLIST, GENERATED_FILE_ALLOWLIST, PLATFORM_ROUTE_HREFS, PLATFORM_ROUTES, PLATFORM_WORKERS, permanentLinkPath, PROJECT_LAYOUT, RESERVED_SLUGS, SHORT_ID_PATTERN, } from './project-layout.js';
1
+ export { ACCESS_CLASSES_ROUTE, GENERATED_DIRECTORY_ALLOWLIST, GENERATED_FILE_ALLOWLIST, markdownProjectionPath, PLATFORM_ROUTE_HREFS, PLATFORM_ROUTES, PLATFORM_WORKERS, permanentLinkPath, PROJECT_LAYOUT, RESERVED_SLUGS, resolvePermanentLink, SHORT_ID_PATTERN, } from './project-layout.js';
2
2
  export { assertGeneratedPathAllowed, isGeneratedPathAllowed, normalizeRepositoryPath, } from './generated-paths.js';
3
3
  export { generatedMarkdownHeader, generatedSourceHeader, } from './ownership-markers.js';
4
4
  export { findProjectRoot, ProjectRootError } from './project-root.js';
@@ -6,4 +6,4 @@ export { EVERY_MEMBER, parseAccessConfiguration, } from './access-configuration.
6
6
  export { CorpusStructureError, EMPTY_CORPUS, parseCorpusStructure, readCorpusStructure, } from './corpus-structure.js';
7
7
  export { ADMINS_CLASS, chainReaders, chainRules, classIdentifiers, computeAccessModel, documentClass, folderChain, folderClass, intersectReaders, MEMBERS_CLASS, nextPublishedReaders, widensReaders, } from './access-classes.js';
8
8
  export { accessFindings } from './access-findings.js';
9
- export { loadSiteConfiguration, parseSiteConfiguration, SiteConfigurationError, } from './site-configuration.js';
9
+ export { fetchCharacterLimit, loadSiteConfiguration, MCP_DEFAULTS, parseSiteConfiguration, SiteConfigurationError, } from './site-configuration.js';
@@ -80,6 +80,16 @@ export declare const PLATFORM_WORKERS: {
80
80
  readonly accessMapAlias: "ctcdocs-access-map";
81
81
  readonly stateBinding: "KB_STATE";
82
82
  readonly directorySchedule: "*/10 * * * *";
83
+ /** The MCP server's OAuth state; the library requires this name (ADR-041). */
84
+ readonly oauthBinding: "OAUTH_KV";
85
+ /** The R2 bucket the Worker publishes each document's Markdown to. */
86
+ readonly documentsBinding: "KB_DOCUMENTS";
87
+ /** The AI Search instance that indexes that bucket. */
88
+ readonly searchBinding: "KB_SEARCH";
89
+ /** How often the site's Worker checks that the bucket matches its build. */
90
+ readonly publishSchedule: "*/5 * * * *";
91
+ /** Lets the OAuth library fetch a client's metadata document safely. */
92
+ readonly oauthCompatibilityFlag: "global_fetch_strictly_public";
83
93
  };
84
94
  /**
85
95
  * Where the Worker tells a signed-in reader which search bundles, beyond the
@@ -100,5 +110,25 @@ export declare const RESERVED_SLUGS: readonly string[];
100
110
  export declare const SHORT_ID_PATTERN: RegExp;
101
111
  /** The site-relative permanent link of an item with this short ID. */
102
112
  export declare function permanentLinkPath(shortId: string): string;
113
+ /**
114
+ * Renders a permanent link as the address it currently leads to.
115
+ *
116
+ * The sync writes a link between two documents as the target's permanent
117
+ * link, `/d/<short ID>/`, so the document holding it never changes when the
118
+ * target is renamed or moved (ADR-022). The redirect map says where each one
119
+ * leads today. Resolving it while the site builds sends a reader straight to
120
+ * the page instead of through a redirect page on every click.
121
+ *
122
+ * A permanent link the map does not know is left as it is: it is either a
123
+ * hand-written link to nothing, which the sync already refuses for generated
124
+ * pages, or one a later build will know.
125
+ */
126
+ export declare function resolvePermanentLink(value: string, redirects: Readonly<Record<string, string>>): string | undefined;
127
+ /**
128
+ * The site path of a page's Markdown version, from its route ID (ADR-010):
129
+ * the one definition, shared by the route that serves it, the pages that link
130
+ * it and the sync that measures it.
131
+ */
132
+ export declare function markdownProjectionPath(slug: string): string;
103
133
  export declare const GENERATED_DIRECTORY_ALLOWLIST: readonly ["src/content/docs/_generated", "src/assets/generated", "src/generated"];
104
134
  export declare const GENERATED_FILE_ALLOWLIST: readonly ["data/sync-manifest.json", "data/docs-index.json", "data/latest-sync-report.json", "data/title-report.json"];
@@ -80,6 +80,16 @@ export const PLATFORM_WORKERS = {
80
80
  accessMapAlias: 'ctcdocs-access-map',
81
81
  stateBinding: 'KB_STATE',
82
82
  directorySchedule: '*/10 * * * *',
83
+ /** The MCP server's OAuth state; the library requires this name (ADR-041). */
84
+ oauthBinding: 'OAUTH_KV',
85
+ /** The R2 bucket the Worker publishes each document's Markdown to. */
86
+ documentsBinding: 'KB_DOCUMENTS',
87
+ /** The AI Search instance that indexes that bucket. */
88
+ searchBinding: 'KB_SEARCH',
89
+ /** How often the site's Worker checks that the bucket matches its build. */
90
+ publishSchedule: '*/5 * * * *',
91
+ /** Lets the OAuth library fetch a client's metadata document safely. */
92
+ oauthCompatibilityFlag: 'global_fetch_strictly_public',
83
93
  };
84
94
  /**
85
95
  * Where the Worker tells a signed-in reader which search bundles, beyond the
@@ -93,8 +103,10 @@ export const PLATFORM_ROUTE_HREFS = {
93
103
  };
94
104
  /**
95
105
  * Top-level addresses the platform serves beside the corpus: the Worker's
96
- * sign-in routes (ADR-038), the search bundles, `/pagefind/` and
97
- * `/pagefind-<class>/` (ADR-039), and the original files under `/assets/`.
106
+ * sign-in and OAuth routes (ADR-038, ADR-041), the search bundles,
107
+ * `/pagefind/` and `/pagefind-<class>/` (ADR-039), and the original files
108
+ * under `/assets/`. The MCP server needs none: it answers `/mcp` exactly
109
+ * (ADR-041), and a page is always served under its trailing slash.
98
110
  */
99
111
  const SERVED_ADDRESSES = ['auth', 'pagefind', 'assets'];
100
112
  export const RESERVED_SLUGS = Object.freeze([
@@ -111,6 +123,38 @@ export const SHORT_ID_PATTERN = /^[0-9a-f]{6,64}$/u;
111
123
  export function permanentLinkPath(shortId) {
112
124
  return `/${PLATFORM_ROUTES.permanentLinks}/${shortId}/`;
113
125
  }
126
+ /**
127
+ * Renders a permanent link as the address it currently leads to.
128
+ *
129
+ * The sync writes a link between two documents as the target's permanent
130
+ * link, `/d/<short ID>/`, so the document holding it never changes when the
131
+ * target is renamed or moved (ADR-022). The redirect map says where each one
132
+ * leads today. Resolving it while the site builds sends a reader straight to
133
+ * the page instead of through a redirect page on every click.
134
+ *
135
+ * A permanent link the map does not know is left as it is: it is either a
136
+ * hand-written link to nothing, which the sync already refuses for generated
137
+ * pages, or one a later build will know.
138
+ */
139
+ export function resolvePermanentLink(value, redirects) {
140
+ const prefix = `/${PLATFORM_ROUTES.permanentLinks}/`;
141
+ if (!value.startsWith(prefix)) {
142
+ return undefined;
143
+ }
144
+ const suffixStart = value.search(/[?#]/u);
145
+ const path = suffixStart < 0 ? value : value.slice(0, suffixStart);
146
+ const suffix = suffixStart < 0 ? '' : value.slice(suffixStart);
147
+ const target = redirects[path.endsWith('/') ? path : `${path}/`];
148
+ return target === undefined ? undefined : `${target}${suffix}`;
149
+ }
150
+ /**
151
+ * The site path of a page's Markdown version, from its route ID (ADR-010):
152
+ * the one definition, shared by the route that serves it, the pages that link
153
+ * it and the sync that measures it.
154
+ */
155
+ export function markdownProjectionPath(slug) {
156
+ return `/${slug}/index.md`;
157
+ }
114
158
  export const GENERATED_DIRECTORY_ALLOWLIST = [
115
159
  PROJECT_LAYOUT.generatedDocumentsDirectory,
116
160
  PROJECT_LAYOUT.generatedAssetsDirectory,
@@ -0,0 +1,33 @@
1
+ export interface PublishedMarkdownInput {
2
+ title: string;
3
+ /**
4
+ * The ownership marker the generated file opens with, which is stripped from
5
+ * the published form. It is passed in rather than read from the project so
6
+ * this stays a pure function of its input.
7
+ */
8
+ ownershipHeader: string;
9
+ sourceUrl: string;
10
+ /** The published PDF, when the page is one's and the site serves it. */
11
+ fileUrl?: string;
12
+ googleModifiedTime: string;
13
+ syncedAt: string;
14
+ contentHash: string;
15
+ body: string;
16
+ stableSlugs: ReadonlySet<string>;
17
+ /**
18
+ * Each permanent link, `/d/<short ID>/`, to the address it leads to. A link
19
+ * between documents is stored as one (ADR-022); the projection names the
20
+ * target's own Markdown instead, as it does for any other corpus link.
21
+ */
22
+ permanentLinks: Readonly<Record<string, string>>;
23
+ }
24
+ /** Where the site serves a published PDF, as its Markdown version names it. */
25
+ export declare function publishedFileUrl(googleFileId: string, file: string): string;
26
+ /**
27
+ * One line of inline Markdown that reads as the text it was given: whitespace,
28
+ * newlines included, collapses to single spaces, and every character that
29
+ * would open emphasis, code, a link, HTML or a heading is escaped. Used for
30
+ * titles, which come from Drive names and may contain any of them.
31
+ */
32
+ export declare function inlineMarkdown(value: string): string;
33
+ export declare function serializePublishedMarkdown(input: PublishedMarkdownInput): string;
@@ -0,0 +1,117 @@
1
+ import * as cheerio from 'cheerio';
2
+ import remarkGfm from 'remark-gfm';
3
+ import remarkParse from 'remark-parse';
4
+ import remarkStringify from 'remark-stringify';
5
+ import { unified } from 'unified';
6
+ import { markdownProjectionPath, resolvePermanentLink, } from './project-layout.js';
7
+ const processor = unified()
8
+ .use(remarkParse)
9
+ .use(remarkGfm)
10
+ .use(remarkStringify, {
11
+ bullet: '-',
12
+ emphasis: '*',
13
+ fences: true,
14
+ listItemIndent: 'one',
15
+ strong: '*',
16
+ });
17
+ function walk(node, visit) {
18
+ if (node.type !== 'root') {
19
+ visit(node);
20
+ }
21
+ if ('children' in node) {
22
+ for (const child of node.children) {
23
+ walk(child, visit);
24
+ }
25
+ }
26
+ }
27
+ function markdownUrl(value, stableSlugs, permanentLinks) {
28
+ const resolved = resolvePermanentLink(value, permanentLinks);
29
+ const link = resolved ?? value;
30
+ if (!link.startsWith('/')) {
31
+ return resolved;
32
+ }
33
+ let url;
34
+ try {
35
+ url = new URL(link, 'https://wiki.invalid');
36
+ }
37
+ catch {
38
+ return resolved;
39
+ }
40
+ const slug = url.pathname.replace(/^\/+|\/+$/gu, '');
41
+ if (!stableSlugs.has(slug)) {
42
+ return resolved;
43
+ }
44
+ return `${markdownProjectionPath(slug)}${url.search}${url.hash}`;
45
+ }
46
+ function rewriteInternalLinks(body, stableSlugs, permanentLinks) {
47
+ const tree = processor.parse(body);
48
+ walk(tree, (node) => {
49
+ if (node.type === 'link') {
50
+ node.url = markdownUrl(node.url, stableSlugs, permanentLinks) ?? node.url;
51
+ return;
52
+ }
53
+ if (node.type !== 'html') {
54
+ return;
55
+ }
56
+ const $ = cheerio.load(node.value, null, false);
57
+ let changed = false;
58
+ $('a[href]').each((_, anchor) => {
59
+ const href = $(anchor).attr('href');
60
+ const rewritten = href
61
+ ? markdownUrl(href, stableSlugs, permanentLinks)
62
+ : undefined;
63
+ if (rewritten) {
64
+ $(anchor).attr('href', rewritten);
65
+ changed = true;
66
+ }
67
+ });
68
+ if (changed) {
69
+ node.value = $.root().html() ?? '';
70
+ }
71
+ });
72
+ return processor.stringify(tree).trimEnd();
73
+ }
74
+ function cleanBody(body, ownershipHeader) {
75
+ const normalized = body.replace(/\r\n?/gu, '\n');
76
+ if (normalized !== ownershipHeader &&
77
+ !normalized.startsWith(`${ownershipHeader}\n`)) {
78
+ throw new Error('Generated Google document is missing the expected ownership marker.');
79
+ }
80
+ return normalized.slice(ownershipHeader.length).trim();
81
+ }
82
+ /** Where the site serves a published PDF, as its Markdown version names it. */
83
+ export function publishedFileUrl(googleFileId, file) {
84
+ return `/assets/generated/${googleFileId}/${file}`;
85
+ }
86
+ function yamlString(value) {
87
+ return JSON.stringify(value);
88
+ }
89
+ /**
90
+ * One line of inline Markdown that reads as the text it was given: whitespace,
91
+ * newlines included, collapses to single spaces, and every character that
92
+ * would open emphasis, code, a link, HTML or a heading is escaped. Used for
93
+ * titles, which come from Drive names and may contain any of them.
94
+ */
95
+ export function inlineMarkdown(value) {
96
+ return value
97
+ .replace(/\s+/gu, ' ')
98
+ .trim()
99
+ .replace(/([\\`*_[\]<>#])/gu, '\\$1');
100
+ }
101
+ export function serializePublishedMarkdown(input) {
102
+ const body = rewriteInternalLinks(cleanBody(input.body, input.ownershipHeader), input.stableSlugs, input.permanentLinks);
103
+ return [
104
+ '---',
105
+ `title: ${yamlString(input.title)}`,
106
+ `source_url: ${yamlString(input.sourceUrl)}`,
107
+ ...(input.fileUrl ? [`file_url: ${yamlString(input.fileUrl)}`] : []),
108
+ `modified_at: ${yamlString(input.googleModifiedTime)}`,
109
+ `synced_at: ${yamlString(input.syncedAt)}`,
110
+ `content_hash: ${yamlString(input.contentHash)}`,
111
+ '---',
112
+ '',
113
+ `# ${inlineMarkdown(input.title)}`,
114
+ '',
115
+ ...(body ? [body, ''] : []),
116
+ ].join('\n');
117
+ }
@@ -94,6 +94,13 @@ export interface SyncConfigurationDefaults {
94
94
  * project sets the line it wants numbers for. Defaults to 2.
95
95
  */
96
96
  readonly largeImageMegabytes: number;
97
+ /**
98
+ * The length, in characters of a document's Markdown body, above which the
99
+ * sync report notes the document as worth splitting (ADR-043). No greater
100
+ * than the characters an assistant reads of a document; defaults to 40,000,
101
+ * or to that limit where it is lower.
102
+ */
103
+ readonly largeDocumentCharacters: number;
97
104
  }
98
105
  export interface NavigationConfiguration {
99
106
  /**
@@ -135,9 +142,63 @@ export type AddressPolicy = 'stable' | 'follow-names';
135
142
  export interface SignInConfiguration {
136
143
  readonly workspaceDomains: readonly string[];
137
144
  }
145
+ /**
146
+ * How the MCP server searches and how much a search returns (ADR-042). Every
147
+ * value has a default, the starting point ADR-042 records, so that a project
148
+ * tunes it against its own corpus without a new version of the platform.
149
+ */
150
+ export interface McpSearchConfiguration {
151
+ /** Chunks asked of AI Search for each query: 1 to 50. */
152
+ readonly chunks: number;
153
+ /**
154
+ * The vector similarity below which AI Search drops a chunk, 0 to 1. It
155
+ * applies to vector similarity alone, so a keyword match far from the
156
+ * question in meaning is dropped too when it is set high.
157
+ */
158
+ readonly vectorThreshold: number;
159
+ /** Whether a keyword match needs every word of the query, or any. */
160
+ readonly keywordMatch: 'and' | 'or';
161
+ /** Neighbouring chunks AI Search adds on each side of a match: 0 to 3. */
162
+ readonly contextChunks: number;
163
+ readonly reranking: {
164
+ readonly enabled: boolean;
165
+ /** An AI Search reranking model. */
166
+ readonly model: string;
167
+ /** The reranking score below which a chunk is dropped, 0 to 1. */
168
+ readonly threshold: number;
169
+ };
170
+ /** Documents a search returns at most; no more than `chunks`. */
171
+ readonly results: number;
172
+ /** Passages each document shows at most. */
173
+ readonly passagesPerResult: number;
174
+ /** Characters of passage text one search returns at most, in all. */
175
+ readonly passageCharacters: number;
176
+ }
177
+ /**
178
+ * The MCP server through which AI assistants read the site as the person who
179
+ * connected them (ADR-041). It needs the Worker's sign-in, so it is accepted
180
+ * only on a deployment whose every environment is private.
181
+ */
182
+ export interface McpConfiguration {
183
+ readonly enabled: boolean;
184
+ readonly search: McpSearchConfiguration;
185
+ /** Characters `fetch` returns at most; a longer document is cut there. */
186
+ readonly fetchCharacters: number;
187
+ }
188
+ /** What `mcp` holds when a project sets only `enabled` (ADR-042). */
189
+ export declare const MCP_DEFAULTS: Omit<McpConfiguration, 'enabled'>;
190
+ /**
191
+ * The characters an assistant reads of a document: the cut `fetch` makes
192
+ * (ADR-042), or its default while the MCP server is off. The build writes
193
+ * this value into the access map and the sync measures documents against it
194
+ * (ADR-043), so the two never derive it apart.
195
+ */
196
+ export declare function fetchCharacterLimit(mcp: McpConfiguration | undefined): number;
138
197
  export interface SiteConfiguration {
139
198
  /** Required to serve a private environment through the platform's Worker. */
140
199
  readonly signIn?: SignInConfiguration;
200
+ /** Absent or disabled, the Worker serves no MCP or OAuth route. */
201
+ readonly mcp?: McpConfiguration;
141
202
  /**
142
203
  * Who may read which folder (ADR-039). Absent, every document is open to
143
204
  * every reader the deployment admits, as before access rules existed.
@@ -15,6 +15,37 @@ import { readFileSync } from 'node:fs';
15
15
  import { resolve } from 'node:path';
16
16
  import { parseAccessConfiguration, } from './access-configuration.js';
17
17
  import { PROJECT_LAYOUT } from './project-layout.js';
18
+ /** What `mcp` holds when a project sets only `enabled` (ADR-042). */
19
+ export const MCP_DEFAULTS = Object.freeze({
20
+ search: Object.freeze({
21
+ chunks: 50,
22
+ vectorThreshold: 0.2,
23
+ keywordMatch: 'or',
24
+ contextChunks: 1,
25
+ reranking: Object.freeze({
26
+ enabled: true,
27
+ model: '@cf/baai/bge-reranker-base',
28
+ threshold: 0,
29
+ }),
30
+ results: 10,
31
+ passagesPerResult: 3,
32
+ passageCharacters: 24_000,
33
+ }),
34
+ fetchCharacters: 100_000,
35
+ });
36
+ /**
37
+ * The characters an assistant reads of a document: the cut `fetch` makes
38
+ * (ADR-042), or its default while the MCP server is off. The build writes
39
+ * this value into the access map and the sync measures documents against it
40
+ * (ADR-043), so the two never derive it apart.
41
+ */
42
+ export function fetchCharacterLimit(mcp) {
43
+ return mcp?.enabled ? mcp.fetchCharacters : MCP_DEFAULTS.fetchCharacters;
44
+ }
45
+ /** About 10,000 tokens of English: a starting value, not a measured one. */
46
+ const LARGE_DOCUMENT_CHARACTERS = 40_000;
47
+ /** A passage of fewer characters than this says too little to judge by. */
48
+ const SHORTEST_PASSAGE = 100;
18
49
  export class SiteConfigurationError extends Error {
19
50
  name = 'SiteConfigurationError';
20
51
  }
@@ -136,6 +167,73 @@ function optionalCount(source, key, path, fallback) {
136
167
  return value;
137
168
  }
138
169
  /** A size above zero, in megabytes; a fraction such as 1.5 is allowed. */
170
+ /** A number between `min` and `max`, whole when `whole`, or the fallback. */
171
+ function optionalNumber(source, key, path, fallback, { min, max, whole }) {
172
+ const value = source[key];
173
+ if (value === undefined) {
174
+ return fallback;
175
+ }
176
+ if (typeof value !== 'number' ||
177
+ !Number.isFinite(value) ||
178
+ (whole && !Number.isInteger(value)) ||
179
+ value < min ||
180
+ (max !== undefined && value > max)) {
181
+ const kind = whole ? 'a whole number' : 'a number';
182
+ fail(path, max === undefined
183
+ ? `must be ${kind} of at least ${min}`
184
+ : `must be ${kind} from ${min} to ${max}`);
185
+ }
186
+ return value;
187
+ }
188
+ /** Fails on a key the section does not define. */
189
+ function knownKeys(source, path, keys) {
190
+ for (const key of Object.keys(source)) {
191
+ if (!keys.includes(key)) {
192
+ fail(`${path}.${key}`, 'is not a known setting');
193
+ }
194
+ }
195
+ }
196
+ function mcpSearch(value) {
197
+ const defaults = MCP_DEFAULTS.search;
198
+ if (value === undefined) {
199
+ return defaults;
200
+ }
201
+ const source = record(value, 'mcp.search');
202
+ knownKeys(source, 'mcp.search', Object.keys(defaults));
203
+ const chunks = optionalNumber(source, 'chunks', 'mcp.search.chunks', defaults.chunks, {
204
+ min: 1,
205
+ max: 50,
206
+ whole: true,
207
+ });
208
+ const keywordMatch = source.keywordMatch ?? defaults.keywordMatch;
209
+ if (keywordMatch !== 'and' && keywordMatch !== 'or') {
210
+ fail('mcp.search.keywordMatch', 'must be "and" or "or"');
211
+ }
212
+ let reranking = defaults.reranking;
213
+ if (source.reranking !== undefined) {
214
+ const rerankingSource = record(source.reranking, 'mcp.search.reranking');
215
+ knownKeys(rerankingSource, 'mcp.search.reranking', Object.keys(reranking));
216
+ reranking = {
217
+ enabled: optionalFlag(rerankingSource, 'enabled', 'mcp.search.reranking.enabled', reranking.enabled),
218
+ model: rerankingSource.model === undefined
219
+ ? reranking.model
220
+ : text(rerankingSource, 'model', 'mcp.search.reranking.model'),
221
+ threshold: optionalNumber(rerankingSource, 'threshold', 'mcp.search.reranking.threshold', reranking.threshold, { min: 0, max: 1, whole: false }),
222
+ };
223
+ }
224
+ // The default, like a set value, is no more than the chunks asked for.
225
+ const results = optionalNumber(source, 'results', 'mcp.search.results', Math.min(defaults.results, chunks), { min: 1, max: chunks, whole: true });
226
+ return {
227
+ chunks,
228
+ vectorThreshold: optionalNumber(source, 'vectorThreshold', 'mcp.search.vectorThreshold', defaults.vectorThreshold, { min: 0, max: 1, whole: false }),
229
+ keywordMatch,
230
+ contextChunks: optionalNumber(source, 'contextChunks', 'mcp.search.contextChunks', defaults.contextChunks, { min: 0, max: 3, whole: true }),
231
+ reranking,
232
+ results,
233
+ passagesPerResult: optionalNumber(source, 'passagesPerResult', 'mcp.search.passagesPerResult', defaults.passagesPerResult, { min: 1, whole: true }),
234
+ passageCharacters: optionalNumber(source, 'passageCharacters', 'mcp.search.passageCharacters', defaults.passageCharacters, { min: results * SHORTEST_PASSAGE, whole: true }),
235
+ };
236
+ }
139
237
  function optionalMegabytes(source, key, path, fallback) {
140
238
  const value = source[key];
141
239
  if (value === undefined) {
@@ -215,6 +313,14 @@ export function parseSiteConfiguration(input) {
215
313
  fail('deployment.workerName', 'must be lowercase letters, digits, and hyphens');
216
314
  }
217
315
  const syncSource = record(root.sync, 'sync');
316
+ // A misspelled optional line would otherwise fall back to its default.
317
+ knownKeys(syncSource, 'sync', [
318
+ 'generatedBy',
319
+ 'commitBotName',
320
+ 'defaultLocale',
321
+ 'largeImageMegabytes',
322
+ 'largeDocumentCharacters',
323
+ ]);
218
324
  const generatedBy = text(syncSource, 'generatedBy', 'sync.generatedBy');
219
325
  // The marker is embedded in an HTML comment in every generated Markdown
220
326
  // file. A comment delimiter inside it would terminate that comment early and
@@ -232,6 +338,7 @@ export function parseSiteConfiguration(input) {
232
338
  environments: environments(deploymentSource),
233
339
  workerName,
234
340
  };
341
+ const publicEnvironment = Object.entries(deployment.environments).find(([, environment]) => environment.visibility === 'public');
235
342
  /*
236
343
  * Rules close folders to some readers, which a public environment cannot
237
344
  * do: everyone may read it. Accepting rules there would promise a boundary
@@ -239,7 +346,6 @@ export function parseSiteConfiguration(input) {
239
346
  */
240
347
  let access;
241
348
  if (root.access !== undefined) {
242
- const publicEnvironment = Object.entries(deployment.environments).find(([, environment]) => environment.visibility === 'public');
243
349
  if (publicEnvironment) {
244
350
  fail('access', `must not be set while the ${publicEnvironment[0]} environment is public`);
245
351
  }
@@ -269,8 +375,29 @@ export function parseSiteConfiguration(input) {
269
375
  }
270
376
  signIn = { workspaceDomains: Object.freeze(parsed) };
271
377
  }
378
+ let mcp;
379
+ if (root.mcp !== undefined) {
380
+ const source = record(root.mcp, 'mcp');
381
+ knownKeys(source, 'mcp', ['enabled', 'search', 'fetchCharacters']);
382
+ const enabled = flag(source, 'enabled', 'mcp.enabled');
383
+ if (enabled) {
384
+ if (!signIn) {
385
+ fail('mcp', 'needs signIn: assistants sign in as the site does');
386
+ }
387
+ if (publicEnvironment) {
388
+ fail('mcp', `must not be enabled while the ${publicEnvironment[0]} environment is public`);
389
+ }
390
+ }
391
+ mcp = {
392
+ enabled,
393
+ search: mcpSearch(source.search),
394
+ fetchCharacters: optionalNumber(source, 'fetchCharacters', 'mcp.fetchCharacters', MCP_DEFAULTS.fetchCharacters, { min: 1_000, whole: true }),
395
+ };
396
+ }
397
+ const fetchLimit = fetchCharacterLimit(mcp);
272
398
  return {
273
399
  ...(signIn ? { signIn } : {}),
400
+ ...(mcp ? { mcp } : {}),
274
401
  ...(access ? { access } : {}),
275
402
  brand,
276
403
  deployment,
@@ -290,6 +417,7 @@ export function parseSiteConfiguration(input) {
290
417
  defaultLocale,
291
418
  generatedBy,
292
419
  largeImageMegabytes: optionalMegabytes(syncSource, 'largeImageMegabytes', 'sync.largeImageMegabytes', 2),
420
+ largeDocumentCharacters: optionalNumber(syncSource, 'largeDocumentCharacters', 'sync.largeDocumentCharacters', Math.min(LARGE_DOCUMENT_CHARACTERS, fetchLimit), { min: 1, max: fetchLimit, whole: true }),
293
421
  },
294
422
  };
295
423
  }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@ctcstack/ctcdocs-core",
3
- "version": "0.16.2",
4
- "description": "Configuration schema, project layout, and generated-path allowlist shared by the CTCDocs platform packages.",
3
+ "version": "0.17.0",
4
+ "description": "Configuration schema, project layout, generated-path allowlist and the published Markdown form shared by the CTCDocs platform packages.",
5
5
  "keywords": [
6
6
  "ctcdocs",
7
7
  "configuration",
@@ -27,6 +27,10 @@
27
27
  ".": {
28
28
  "types": "./dist/index.d.ts",
29
29
  "default": "./dist/index.js"
30
+ },
31
+ "./published-markdown": {
32
+ "types": "./dist/published-markdown.d.ts",
33
+ "default": "./dist/published-markdown.js"
30
34
  }
31
35
  },
32
36
  "files": [
@@ -37,7 +41,15 @@
37
41
  "engines": {
38
42
  "node": ">=22.12.0"
39
43
  },
44
+ "dependencies": {
45
+ "cheerio": "1.2.0",
46
+ "remark-gfm": "4.0.1",
47
+ "remark-parse": "11.0.0",
48
+ "remark-stringify": "11.0.0",
49
+ "unified": "11.0.5"
50
+ },
40
51
  "devDependencies": {
52
+ "@types/mdast": "4.0.4",
41
53
  "@types/node": "22.20.1",
42
54
  "fast-check": "4.10.2",
43
55
  "typescript": "6.0.3",
@@ -33,6 +33,10 @@ export interface CorpusDocument {
33
33
  * on its new chain to be confirmed.
34
34
  */
35
35
  readonly publishedReaders?: '*' | readonly string[];
36
+ /** The permanent short ID (ADR-022), the title and Drive's modified time. */
37
+ readonly shortId?: string;
38
+ readonly title?: string;
39
+ readonly modified?: string;
36
40
  }
37
41
 
38
42
  export interface CorpusStructure {
@@ -101,11 +105,22 @@ export function parseCorpusStructure(manifest: unknown): CorpusStructure {
101
105
  const documents = new Map<string, CorpusDocument>();
102
106
  for (const [id, raw] of entries(source.documents, 'documents')) {
103
107
  const path = `documents.${id}`;
104
- const published = (raw as Record<string, unknown>).publishedReaders;
108
+ const record = raw as Record<string, unknown>;
109
+ const published = record.publishedReaders;
110
+ const optional = (key: string) =>
111
+ typeof record[key] === 'string' && record[key].length > 0
112
+ ? record[key]
113
+ : undefined;
114
+ const shortId = optional('shortId');
115
+ const title = optional('displayTitle');
116
+ const modified = optional('googleModifiedTime');
105
117
  documents.set(id, {
106
118
  id,
107
119
  parentId: field(raw, 'googleParentId', path, true),
108
120
  slug: field(raw, 'stableSlug', path) ?? '',
121
+ ...(shortId ? { shortId } : {}),
122
+ ...(title ? { title } : {}),
123
+ ...(modified ? { modified } : {}),
109
124
  ...(published === '*' ||
110
125
  (Array.isArray(published) &&
111
126
  published.every((group) => typeof group === 'string'))
package/src/index.ts CHANGED
@@ -2,12 +2,14 @@ export {
2
2
  ACCESS_CLASSES_ROUTE,
3
3
  GENERATED_DIRECTORY_ALLOWLIST,
4
4
  GENERATED_FILE_ALLOWLIST,
5
+ markdownProjectionPath,
5
6
  PLATFORM_ROUTE_HREFS,
6
7
  PLATFORM_ROUTES,
7
8
  PLATFORM_WORKERS,
8
9
  permanentLinkPath,
9
10
  PROJECT_LAYOUT,
10
11
  RESERVED_SLUGS,
12
+ resolvePermanentLink,
11
13
  SHORT_ID_PATTERN,
12
14
  } from './project-layout.js';
13
15
  export {
@@ -63,7 +65,9 @@ export type {
63
65
  export { accessFindings } from './access-findings.js';
64
66
  export type { AccessFinding } from './access-findings.js';
65
67
  export {
68
+ fetchCharacterLimit,
66
69
  loadSiteConfiguration,
70
+ MCP_DEFAULTS,
67
71
  parseSiteConfiguration,
68
72
  SiteConfigurationError,
69
73
  } from './site-configuration.js';
@@ -75,6 +79,8 @@ export type {
75
79
  DeploymentEnvironmentConfigurations,
76
80
  DeploymentVisibility,
77
81
  HomeConfiguration,
82
+ McpConfiguration,
83
+ McpSearchConfiguration,
78
84
  NavigationConfiguration,
79
85
  SignInConfiguration,
80
86
  SiteConfiguration,
@@ -82,6 +82,16 @@ export const PLATFORM_WORKERS = {
82
82
  accessMapAlias: 'ctcdocs-access-map',
83
83
  stateBinding: 'KB_STATE',
84
84
  directorySchedule: '*/10 * * * *',
85
+ /** The MCP server's OAuth state; the library requires this name (ADR-041). */
86
+ oauthBinding: 'OAUTH_KV',
87
+ /** The R2 bucket the Worker publishes each document's Markdown to. */
88
+ documentsBinding: 'KB_DOCUMENTS',
89
+ /** The AI Search instance that indexes that bucket. */
90
+ searchBinding: 'KB_SEARCH',
91
+ /** How often the site's Worker checks that the bucket matches its build. */
92
+ publishSchedule: '*/5 * * * *',
93
+ /** Lets the OAuth library fetch a client's metadata document safely. */
94
+ oauthCompatibilityFlag: 'global_fetch_strictly_public',
85
95
  } as const;
86
96
 
87
97
  /**
@@ -98,8 +108,10 @@ export const PLATFORM_ROUTE_HREFS = {
98
108
 
99
109
  /**
100
110
  * Top-level addresses the platform serves beside the corpus: the Worker's
101
- * sign-in routes (ADR-038), the search bundles, `/pagefind/` and
102
- * `/pagefind-<class>/` (ADR-039), and the original files under `/assets/`.
111
+ * sign-in and OAuth routes (ADR-038, ADR-041), the search bundles,
112
+ * `/pagefind/` and `/pagefind-<class>/` (ADR-039), and the original files
113
+ * under `/assets/`. The MCP server needs none: it answers `/mcp` exactly
114
+ * (ADR-041), and a page is always served under its trailing slash.
103
115
  */
104
116
  const SERVED_ADDRESSES = ['auth', 'pagefind', 'assets'] as const;
105
117
 
@@ -120,6 +132,43 @@ export function permanentLinkPath(shortId: string): string {
120
132
  return `/${PLATFORM_ROUTES.permanentLinks}/${shortId}/`;
121
133
  }
122
134
 
135
+ /**
136
+ * Renders a permanent link as the address it currently leads to.
137
+ *
138
+ * The sync writes a link between two documents as the target's permanent
139
+ * link, `/d/<short ID>/`, so the document holding it never changes when the
140
+ * target is renamed or moved (ADR-022). The redirect map says where each one
141
+ * leads today. Resolving it while the site builds sends a reader straight to
142
+ * the page instead of through a redirect page on every click.
143
+ *
144
+ * A permanent link the map does not know is left as it is: it is either a
145
+ * hand-written link to nothing, which the sync already refuses for generated
146
+ * pages, or one a later build will know.
147
+ */
148
+ export function resolvePermanentLink(
149
+ value: string,
150
+ redirects: Readonly<Record<string, string>>,
151
+ ): string | undefined {
152
+ const prefix = `/${PLATFORM_ROUTES.permanentLinks}/`;
153
+ if (!value.startsWith(prefix)) {
154
+ return undefined;
155
+ }
156
+ const suffixStart = value.search(/[?#]/u);
157
+ const path = suffixStart < 0 ? value : value.slice(0, suffixStart);
158
+ const suffix = suffixStart < 0 ? '' : value.slice(suffixStart);
159
+ const target = redirects[path.endsWith('/') ? path : `${path}/`];
160
+ return target === undefined ? undefined : `${target}${suffix}`;
161
+ }
162
+
163
+ /**
164
+ * The site path of a page's Markdown version, from its route ID (ADR-010):
165
+ * the one definition, shared by the route that serves it, the pages that link
166
+ * it and the sync that measures it.
167
+ */
168
+ export function markdownProjectionPath(slug: string): string {
169
+ return `/${slug}/index.md`;
170
+ }
171
+
123
172
  export const GENERATED_DIRECTORY_ALLOWLIST = [
124
173
  PROJECT_LAYOUT.generatedDocumentsDirectory,
125
174
  PROJECT_LAYOUT.generatedAssetsDirectory,
@@ -0,0 +1,178 @@
1
+ import type { Root, RootContent } from 'mdast';
2
+ import * as cheerio from 'cheerio';
3
+ import remarkGfm from 'remark-gfm';
4
+ import remarkParse from 'remark-parse';
5
+ import remarkStringify from 'remark-stringify';
6
+ import { unified } from 'unified';
7
+
8
+ import {
9
+ markdownProjectionPath,
10
+ resolvePermanentLink,
11
+ } from './project-layout.js';
12
+
13
+ const processor = unified()
14
+ .use(remarkParse)
15
+ .use(remarkGfm)
16
+ .use(remarkStringify, {
17
+ bullet: '-',
18
+ emphasis: '*',
19
+ fences: true,
20
+ listItemIndent: 'one',
21
+ strong: '*',
22
+ });
23
+
24
+ export interface PublishedMarkdownInput {
25
+ title: string;
26
+ /**
27
+ * The ownership marker the generated file opens with, which is stripped from
28
+ * the published form. It is passed in rather than read from the project so
29
+ * this stays a pure function of its input.
30
+ */
31
+ ownershipHeader: string;
32
+ sourceUrl: string;
33
+ /** The published PDF, when the page is one's and the site serves it. */
34
+ fileUrl?: string;
35
+ googleModifiedTime: string;
36
+ syncedAt: string;
37
+ contentHash: string;
38
+ body: string;
39
+ stableSlugs: ReadonlySet<string>;
40
+ /**
41
+ * Each permanent link, `/d/<short ID>/`, to the address it leads to. A link
42
+ * between documents is stored as one (ADR-022); the projection names the
43
+ * target's own Markdown instead, as it does for any other corpus link.
44
+ */
45
+ permanentLinks: Readonly<Record<string, string>>;
46
+ }
47
+
48
+ function walk(
49
+ node: Root | RootContent,
50
+ visit: (node: RootContent) => void,
51
+ ): void {
52
+ if (node.type !== 'root') {
53
+ visit(node);
54
+ }
55
+ if ('children' in node) {
56
+ for (const child of node.children) {
57
+ walk(child, visit);
58
+ }
59
+ }
60
+ }
61
+
62
+ function markdownUrl(
63
+ value: string,
64
+ stableSlugs: ReadonlySet<string>,
65
+ permanentLinks: Readonly<Record<string, string>>,
66
+ ): string | undefined {
67
+ const resolved = resolvePermanentLink(value, permanentLinks);
68
+ const link = resolved ?? value;
69
+ if (!link.startsWith('/')) {
70
+ return resolved;
71
+ }
72
+
73
+ let url: URL;
74
+ try {
75
+ url = new URL(link, 'https://wiki.invalid');
76
+ } catch {
77
+ return resolved;
78
+ }
79
+ const slug = url.pathname.replace(/^\/+|\/+$/gu, '');
80
+ if (!stableSlugs.has(slug)) {
81
+ return resolved;
82
+ }
83
+ return `${markdownProjectionPath(slug)}${url.search}${url.hash}`;
84
+ }
85
+
86
+ function rewriteInternalLinks(
87
+ body: string,
88
+ stableSlugs: ReadonlySet<string>,
89
+ permanentLinks: Readonly<Record<string, string>>,
90
+ ): string {
91
+ const tree = processor.parse(body) as Root;
92
+ walk(tree, (node) => {
93
+ if (node.type === 'link') {
94
+ node.url = markdownUrl(node.url, stableSlugs, permanentLinks) ?? node.url;
95
+ return;
96
+ }
97
+ if (node.type !== 'html') {
98
+ return;
99
+ }
100
+
101
+ const $ = cheerio.load(node.value, null, false);
102
+ let changed = false;
103
+ $('a[href]').each((_, anchor) => {
104
+ const href = $(anchor).attr('href');
105
+ const rewritten = href
106
+ ? markdownUrl(href, stableSlugs, permanentLinks)
107
+ : undefined;
108
+ if (rewritten) {
109
+ $(anchor).attr('href', rewritten);
110
+ changed = true;
111
+ }
112
+ });
113
+ if (changed) {
114
+ node.value = $.root().html() ?? '';
115
+ }
116
+ });
117
+
118
+ return processor.stringify(tree).trimEnd();
119
+ }
120
+
121
+ function cleanBody(body: string, ownershipHeader: string): string {
122
+ const normalized = body.replace(/\r\n?/gu, '\n');
123
+ if (
124
+ normalized !== ownershipHeader &&
125
+ !normalized.startsWith(`${ownershipHeader}\n`)
126
+ ) {
127
+ throw new Error(
128
+ 'Generated Google document is missing the expected ownership marker.',
129
+ );
130
+ }
131
+ return normalized.slice(ownershipHeader.length).trim();
132
+ }
133
+
134
+ /** Where the site serves a published PDF, as its Markdown version names it. */
135
+ export function publishedFileUrl(googleFileId: string, file: string): string {
136
+ return `/assets/generated/${googleFileId}/${file}`;
137
+ }
138
+
139
+ function yamlString(value: string): string {
140
+ return JSON.stringify(value);
141
+ }
142
+
143
+ /**
144
+ * One line of inline Markdown that reads as the text it was given: whitespace,
145
+ * newlines included, collapses to single spaces, and every character that
146
+ * would open emphasis, code, a link, HTML or a heading is escaped. Used for
147
+ * titles, which come from Drive names and may contain any of them.
148
+ */
149
+ export function inlineMarkdown(value: string): string {
150
+ return value
151
+ .replace(/\s+/gu, ' ')
152
+ .trim()
153
+ .replace(/([\\`*_[\]<>#])/gu, '\\$1');
154
+ }
155
+
156
+ export function serializePublishedMarkdown(
157
+ input: PublishedMarkdownInput,
158
+ ): string {
159
+ const body = rewriteInternalLinks(
160
+ cleanBody(input.body, input.ownershipHeader),
161
+ input.stableSlugs,
162
+ input.permanentLinks,
163
+ );
164
+ return [
165
+ '---',
166
+ `title: ${yamlString(input.title)}`,
167
+ `source_url: ${yamlString(input.sourceUrl)}`,
168
+ ...(input.fileUrl ? [`file_url: ${yamlString(input.fileUrl)}`] : []),
169
+ `modified_at: ${yamlString(input.googleModifiedTime)}`,
170
+ `synced_at: ${yamlString(input.syncedAt)}`,
171
+ `content_hash: ${yamlString(input.contentHash)}`,
172
+ '---',
173
+ '',
174
+ `# ${inlineMarkdown(input.title)}`,
175
+ '',
176
+ ...(body ? [body, ''] : []),
177
+ ].join('\n');
178
+ }
@@ -121,6 +121,13 @@ export interface SyncConfigurationDefaults {
121
121
  * project sets the line it wants numbers for. Defaults to 2.
122
122
  */
123
123
  readonly largeImageMegabytes: number;
124
+ /**
125
+ * The length, in characters of a document's Markdown body, above which the
126
+ * sync report notes the document as worth splitting (ADR-043). No greater
127
+ * than the characters an assistant reads of a document; defaults to 40,000,
128
+ * or to that limit where it is lower.
129
+ */
130
+ readonly largeDocumentCharacters: number;
124
131
  }
125
132
 
126
133
  export interface NavigationConfiguration {
@@ -166,9 +173,91 @@ export interface SignInConfiguration {
166
173
  readonly workspaceDomains: readonly string[];
167
174
  }
168
175
 
176
+ /**
177
+ * How the MCP server searches and how much a search returns (ADR-042). Every
178
+ * value has a default, the starting point ADR-042 records, so that a project
179
+ * tunes it against its own corpus without a new version of the platform.
180
+ */
181
+ export interface McpSearchConfiguration {
182
+ /** Chunks asked of AI Search for each query: 1 to 50. */
183
+ readonly chunks: number;
184
+ /**
185
+ * The vector similarity below which AI Search drops a chunk, 0 to 1. It
186
+ * applies to vector similarity alone, so a keyword match far from the
187
+ * question in meaning is dropped too when it is set high.
188
+ */
189
+ readonly vectorThreshold: number;
190
+ /** Whether a keyword match needs every word of the query, or any. */
191
+ readonly keywordMatch: 'and' | 'or';
192
+ /** Neighbouring chunks AI Search adds on each side of a match: 0 to 3. */
193
+ readonly contextChunks: number;
194
+ readonly reranking: {
195
+ readonly enabled: boolean;
196
+ /** An AI Search reranking model. */
197
+ readonly model: string;
198
+ /** The reranking score below which a chunk is dropped, 0 to 1. */
199
+ readonly threshold: number;
200
+ };
201
+ /** Documents a search returns at most; no more than `chunks`. */
202
+ readonly results: number;
203
+ /** Passages each document shows at most. */
204
+ readonly passagesPerResult: number;
205
+ /** Characters of passage text one search returns at most, in all. */
206
+ readonly passageCharacters: number;
207
+ }
208
+
209
+ /**
210
+ * The MCP server through which AI assistants read the site as the person who
211
+ * connected them (ADR-041). It needs the Worker's sign-in, so it is accepted
212
+ * only on a deployment whose every environment is private.
213
+ */
214
+ export interface McpConfiguration {
215
+ readonly enabled: boolean;
216
+ readonly search: McpSearchConfiguration;
217
+ /** Characters `fetch` returns at most; a longer document is cut there. */
218
+ readonly fetchCharacters: number;
219
+ }
220
+
221
+ /** What `mcp` holds when a project sets only `enabled` (ADR-042). */
222
+ export const MCP_DEFAULTS: Omit<McpConfiguration, 'enabled'> = Object.freeze({
223
+ search: Object.freeze({
224
+ chunks: 50,
225
+ vectorThreshold: 0.2,
226
+ keywordMatch: 'or',
227
+ contextChunks: 1,
228
+ reranking: Object.freeze({
229
+ enabled: true,
230
+ model: '@cf/baai/bge-reranker-base',
231
+ threshold: 0,
232
+ }),
233
+ results: 10,
234
+ passagesPerResult: 3,
235
+ passageCharacters: 24_000,
236
+ }),
237
+ fetchCharacters: 100_000,
238
+ });
239
+
240
+ /**
241
+ * The characters an assistant reads of a document: the cut `fetch` makes
242
+ * (ADR-042), or its default while the MCP server is off. The build writes
243
+ * this value into the access map and the sync measures documents against it
244
+ * (ADR-043), so the two never derive it apart.
245
+ */
246
+ export function fetchCharacterLimit(mcp: McpConfiguration | undefined): number {
247
+ return mcp?.enabled ? mcp.fetchCharacters : MCP_DEFAULTS.fetchCharacters;
248
+ }
249
+
250
+ /** About 10,000 tokens of English: a starting value, not a measured one. */
251
+ const LARGE_DOCUMENT_CHARACTERS = 40_000;
252
+
253
+ /** A passage of fewer characters than this says too little to judge by. */
254
+ const SHORTEST_PASSAGE = 100;
255
+
169
256
  export interface SiteConfiguration {
170
257
  /** Required to serve a private environment through the platform's Worker. */
171
258
  readonly signIn?: SignInConfiguration;
259
+ /** Absent or disabled, the Worker serves no MCP or OAuth route. */
260
+ readonly mcp?: McpConfiguration;
172
261
  /**
173
262
  * Who may read which folder (ADR-039). Absent, every document is open to
174
263
  * every reader the deployment admits, as before access rules existed.
@@ -350,6 +439,139 @@ function optionalCount(
350
439
  }
351
440
 
352
441
  /** A size above zero, in megabytes; a fraction such as 1.5 is allowed. */
442
+ /** A number between `min` and `max`, whole when `whole`, or the fallback. */
443
+ function optionalNumber(
444
+ source: Record<string, unknown>,
445
+ key: string,
446
+ path: string,
447
+ fallback: number,
448
+ { min, max, whole }: { min: number; max?: number; whole: boolean },
449
+ ): number {
450
+ const value = source[key];
451
+ if (value === undefined) {
452
+ return fallback;
453
+ }
454
+ if (
455
+ typeof value !== 'number' ||
456
+ !Number.isFinite(value) ||
457
+ (whole && !Number.isInteger(value)) ||
458
+ value < min ||
459
+ (max !== undefined && value > max)
460
+ ) {
461
+ const kind = whole ? 'a whole number' : 'a number';
462
+ fail(
463
+ path,
464
+ max === undefined
465
+ ? `must be ${kind} of at least ${min}`
466
+ : `must be ${kind} from ${min} to ${max}`,
467
+ );
468
+ }
469
+ return value;
470
+ }
471
+
472
+ /** Fails on a key the section does not define. */
473
+ function knownKeys(
474
+ source: Record<string, unknown>,
475
+ path: string,
476
+ keys: readonly string[],
477
+ ): void {
478
+ for (const key of Object.keys(source)) {
479
+ if (!keys.includes(key)) {
480
+ fail(`${path}.${key}`, 'is not a known setting');
481
+ }
482
+ }
483
+ }
484
+
485
+ function mcpSearch(value: unknown): McpSearchConfiguration {
486
+ const defaults = MCP_DEFAULTS.search;
487
+ if (value === undefined) {
488
+ return defaults;
489
+ }
490
+ const source = record(value, 'mcp.search');
491
+ knownKeys(source, 'mcp.search', Object.keys(defaults));
492
+ const chunks = optionalNumber(
493
+ source,
494
+ 'chunks',
495
+ 'mcp.search.chunks',
496
+ defaults.chunks,
497
+ {
498
+ min: 1,
499
+ max: 50,
500
+ whole: true,
501
+ },
502
+ );
503
+ const keywordMatch = source.keywordMatch ?? defaults.keywordMatch;
504
+ if (keywordMatch !== 'and' && keywordMatch !== 'or') {
505
+ fail('mcp.search.keywordMatch', 'must be "and" or "or"');
506
+ }
507
+ let reranking = defaults.reranking;
508
+ if (source.reranking !== undefined) {
509
+ const rerankingSource = record(source.reranking, 'mcp.search.reranking');
510
+ knownKeys(rerankingSource, 'mcp.search.reranking', Object.keys(reranking));
511
+ reranking = {
512
+ enabled: optionalFlag(
513
+ rerankingSource,
514
+ 'enabled',
515
+ 'mcp.search.reranking.enabled',
516
+ reranking.enabled,
517
+ ),
518
+ model:
519
+ rerankingSource.model === undefined
520
+ ? reranking.model
521
+ : text(rerankingSource, 'model', 'mcp.search.reranking.model'),
522
+ threshold: optionalNumber(
523
+ rerankingSource,
524
+ 'threshold',
525
+ 'mcp.search.reranking.threshold',
526
+ reranking.threshold,
527
+ { min: 0, max: 1, whole: false },
528
+ ),
529
+ };
530
+ }
531
+ // The default, like a set value, is no more than the chunks asked for.
532
+ const results = optionalNumber(
533
+ source,
534
+ 'results',
535
+ 'mcp.search.results',
536
+ Math.min(defaults.results, chunks),
537
+ { min: 1, max: chunks, whole: true },
538
+ );
539
+ return {
540
+ chunks,
541
+ vectorThreshold: optionalNumber(
542
+ source,
543
+ 'vectorThreshold',
544
+ 'mcp.search.vectorThreshold',
545
+ defaults.vectorThreshold,
546
+ { min: 0, max: 1, whole: false },
547
+ ),
548
+ keywordMatch,
549
+ contextChunks: optionalNumber(
550
+ source,
551
+ 'contextChunks',
552
+ 'mcp.search.contextChunks',
553
+ defaults.contextChunks,
554
+ { min: 0, max: 3, whole: true },
555
+ ),
556
+ reranking,
557
+ results,
558
+ passagesPerResult: optionalNumber(
559
+ source,
560
+ 'passagesPerResult',
561
+ 'mcp.search.passagesPerResult',
562
+ defaults.passagesPerResult,
563
+ { min: 1, whole: true },
564
+ ),
565
+ passageCharacters: optionalNumber(
566
+ source,
567
+ 'passageCharacters',
568
+ 'mcp.search.passageCharacters',
569
+ defaults.passageCharacters,
570
+ { min: results * SHORTEST_PASSAGE, whole: true },
571
+ ),
572
+ };
573
+ }
574
+
353
575
  function optionalMegabytes(
354
576
  source: Record<string, unknown>,
355
577
  key: string,
@@ -467,6 +689,14 @@ export function parseSiteConfiguration(input: unknown): SiteConfiguration {
467
689
  }
468
690
 
469
691
  const syncSource = record(root.sync, 'sync');
692
+ // A misspelled optional line would otherwise fall back to its default.
693
+ knownKeys(syncSource, 'sync', [
694
+ 'generatedBy',
695
+ 'commitBotName',
696
+ 'defaultLocale',
697
+ 'largeImageMegabytes',
698
+ 'largeDocumentCharacters',
699
+ ]);
470
700
  const generatedBy = text(syncSource, 'generatedBy', 'sync.generatedBy');
471
701
  // The marker is embedded in an HTML comment in every generated Markdown
472
702
  // file. A comment delimiter inside it would terminate that comment early and
@@ -486,6 +716,10 @@ export function parseSiteConfiguration(input: unknown): SiteConfiguration {
486
716
  workerName,
487
717
  };
488
718
 
719
+ const publicEnvironment = Object.entries(deployment.environments).find(
720
+ ([, environment]) => environment.visibility === 'public',
721
+ );
722
+
489
723
  /*
490
724
  * Rules close folders to some readers, which a public environment cannot
491
725
  * do: everyone may read it. Accepting rules there would promise a boundary
@@ -493,9 +727,6 @@ export function parseSiteConfiguration(input: unknown): SiteConfiguration {
493
727
  */
494
728
  let access: AccessConfiguration | undefined;
495
729
  if (root.access !== undefined) {
496
- const publicEnvironment = Object.entries(deployment.environments).find(
497
- ([, environment]) => environment.visibility === 'public',
498
- );
499
730
  if (publicEnvironment) {
500
731
  fail(
501
732
  'access',
@@ -538,8 +769,39 @@ export function parseSiteConfiguration(input: unknown): SiteConfiguration {
538
769
  signIn = { workspaceDomains: Object.freeze(parsed) };
539
770
  }
540
771
 
772
+ let mcp: McpConfiguration | undefined;
773
+ if (root.mcp !== undefined) {
774
+ const source = record(root.mcp, 'mcp');
775
+ knownKeys(source, 'mcp', ['enabled', 'search', 'fetchCharacters']);
776
+ const enabled = flag(source, 'enabled', 'mcp.enabled');
777
+ if (enabled) {
778
+ if (!signIn) {
779
+ fail('mcp', 'needs signIn: assistants sign in as the site does');
780
+ }
781
+ if (publicEnvironment) {
782
+ fail(
783
+ 'mcp',
784
+ `must not be enabled while the ${publicEnvironment[0]} environment is public`,
785
+ );
786
+ }
787
+ }
788
+ mcp = {
789
+ enabled,
790
+ search: mcpSearch(source.search),
791
+ fetchCharacters: optionalNumber(
792
+ source,
793
+ 'fetchCharacters',
794
+ 'mcp.fetchCharacters',
795
+ MCP_DEFAULTS.fetchCharacters,
796
+ { min: 1_000, whole: true },
797
+ ),
798
+ };
799
+ }
800
+ const fetchLimit = fetchCharacterLimit(mcp);
801
+
541
802
  return {
542
803
  ...(signIn ? { signIn } : {}),
804
+ ...(mcp ? { mcp } : {}),
543
805
  ...(access ? { access } : {}),
544
806
  brand,
545
807
  deployment,
@@ -590,6 +852,13 @@ export function parseSiteConfiguration(input: unknown): SiteConfiguration {
590
852
  'sync.largeImageMegabytes',
591
853
  2,
592
854
  ),
855
+ largeDocumentCharacters: optionalNumber(
856
+ syncSource,
857
+ 'largeDocumentCharacters',
858
+ 'sync.largeDocumentCharacters',
859
+ Math.min(LARGE_DOCUMENT_CHARACTERS, fetchLimit),
860
+ { min: 1, max: fetchLimit, whole: true },
861
+ ),
593
862
  },
594
863
  };
595
864
  }