@hashicorp/design-system-mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +375 -0
- package/README.md +83 -0
- package/dist/catalog/loader.js +99 -0
- package/dist/catalog/normalize.js +7 -0
- package/dist/catalog/project-root.js +47 -0
- package/dist/index.js +107 -0
- package/dist/prompts/index.js +10 -0
- package/dist/prompts/types.js +5 -0
- package/dist/resources/completions.js +30 -0
- package/dist/resources/components/constants.js +6 -0
- package/dist/resources/components/get-component-by-name.js +54 -0
- package/dist/resources/components/get-components.js +33 -0
- package/dist/resources/components/index.js +11 -0
- package/dist/resources/components/utils.js +39 -0
- package/dist/resources/define-resource.js +35 -0
- package/dist/resources/hds-icons/constants.js +6 -0
- package/dist/resources/hds-icons/get-icon-by-name.js +48 -0
- package/dist/resources/hds-icons/get-icons.js +33 -0
- package/dist/resources/hds-icons/index.js +11 -0
- package/dist/resources/hds-icons/utils.js +26 -0
- package/dist/resources/index.js +23 -0
- package/dist/resources/responses.js +34 -0
- package/dist/resources/tokens/constants.js +2 -0
- package/dist/resources/tokens/get-token-by-key.js +51 -0
- package/dist/resources/tokens/get-tokens.js +33 -0
- package/dist/resources/tokens/index.js +11 -0
- package/dist/resources/tokens/utils.js +15 -0
- package/dist/resources/types.js +5 -0
- package/dist/resources/uri.js +15 -0
- package/dist/stores/components/index.js +114 -0
- package/dist/stores/components/lookup.js +55 -0
- package/dist/stores/components/schema.js +37 -0
- package/dist/stores/docs/constants.js +163 -0
- package/dist/stores/docs/index.js +322 -0
- package/dist/stores/docs/lookup.js +148 -0
- package/dist/stores/docs/schema.js +60 -0
- package/dist/stores/docs/search-index.js +102 -0
- package/dist/stores/docs/tokenize.js +52 -0
- package/dist/stores/hds-icons/index.js +115 -0
- package/dist/stores/hds-icons/lookup.js +43 -0
- package/dist/stores/hds-icons/schema.js +31 -0
- package/dist/stores/rank.js +129 -0
- package/dist/stores/tokens/index.js +105 -0
- package/dist/stores/tokens/lookup.js +65 -0
- package/dist/stores/tokens/schema.js +42 -0
- package/dist/stores/types.js +5 -0
- package/dist/tools/components/constants.js +15 -0
- package/dist/tools/components/get-component.js +98 -0
- package/dist/tools/components/index.js +11 -0
- package/dist/tools/components/search-components.js +75 -0
- package/dist/tools/components/utils.js +42 -0
- package/dist/tools/constants.js +7 -0
- package/dist/tools/define-tool.js +17 -0
- package/dist/tools/docs/constants.js +33 -0
- package/dist/tools/docs/index.js +8 -0
- package/dist/tools/docs/read-doc.js +101 -0
- package/dist/tools/docs/search-docs.js +143 -0
- package/dist/tools/docs/utils.js +82 -0
- package/dist/tools/hds-icons/constants.js +5 -0
- package/dist/tools/hds-icons/index.js +7 -0
- package/dist/tools/hds-icons/search-icons.js +123 -0
- package/dist/tools/index.js +19 -0
- package/dist/tools/output-schema.js +11 -0
- package/dist/tools/responses.js +52 -0
- package/dist/tools/search.js +36 -0
- package/dist/tools/tokens/constants.js +5 -0
- package/dist/tools/tokens/index.js +7 -0
- package/dist/tools/tokens/search-tokens.js +144 -0
- package/dist/tools/types.js +5 -0
- package/dist/types.js +5 -0
- package/docs-catalog.json +39778 -0
- package/package.json +62 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright IBM Corp. 2021, 2026
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
*/
|
|
5
|
+
import { normalizeLookupValue } from "../../catalog/normalize.js";
|
|
6
|
+
import { DEFAULT_CATALOG_SOURCE, createCatalogLoader, } from "../../catalog/loader.js";
|
|
7
|
+
import { getComponentLookupKeys, normalizeComponentName, toComponentRecord, toComponentSummary, } from "./lookup.js";
|
|
8
|
+
import { componentCatalogSchema } from "./schema.js";
|
|
9
|
+
import { searchRanked, getCommonPrefixLength } from "../rank.js";
|
|
10
|
+
const toSearchBlob = (component) => {
|
|
11
|
+
return [
|
|
12
|
+
component.name,
|
|
13
|
+
component.name.replaceAll("::", ""),
|
|
14
|
+
component.modulePath,
|
|
15
|
+
component.modulePath.replaceAll(/[/-]/gu, " "),
|
|
16
|
+
component.docsPath ?? "",
|
|
17
|
+
]
|
|
18
|
+
.join(" ")
|
|
19
|
+
.toLowerCase();
|
|
20
|
+
};
|
|
21
|
+
// the four ways a caller writes a component: template invocation, bare name, imported class
|
|
22
|
+
// and module path
|
|
23
|
+
const toRankable = (component) => {
|
|
24
|
+
return {
|
|
25
|
+
identities: [
|
|
26
|
+
normalizeLookupValue(component.name),
|
|
27
|
+
normalizeComponentName(component.name),
|
|
28
|
+
normalizeLookupValue(component.name.replaceAll("::", "")),
|
|
29
|
+
normalizeLookupValue(component.modulePath),
|
|
30
|
+
],
|
|
31
|
+
blob: toSearchBlob(component),
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
export const parseComponentCatalog = (value) => {
|
|
35
|
+
return componentCatalogSchema.parse(value);
|
|
36
|
+
};
|
|
37
|
+
export const createComponentCatalogStore = (catalog, source = DEFAULT_CATALOG_SOURCE) => {
|
|
38
|
+
const componentRecords = catalog.components.map((entry) => toComponentRecord(entry, catalog.valueSets ?? {}));
|
|
39
|
+
const componentLookup = new Map();
|
|
40
|
+
for (const [index, entry] of catalog.components.entries()) {
|
|
41
|
+
const component = componentRecords[index];
|
|
42
|
+
if (component === undefined) {
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
for (const key of getComponentLookupKeys(entry)) {
|
|
47
|
+
if (!componentLookup.has(key)) {
|
|
48
|
+
componentLookup.set(key, component);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const searchComponents = ({ query, limit }) => {
|
|
54
|
+
if (normalizeLookupValue(query).length === 0) {
|
|
55
|
+
return { totalMatches: 0, hits: [] };
|
|
56
|
+
}
|
|
57
|
+
const { totalMatches, hits } = searchRanked({
|
|
58
|
+
records: componentRecords,
|
|
59
|
+
query,
|
|
60
|
+
limit,
|
|
61
|
+
toRankable,
|
|
62
|
+
});
|
|
63
|
+
return {
|
|
64
|
+
totalMatches,
|
|
65
|
+
hits: hits.map((component) => toComponentSummary(component)),
|
|
66
|
+
};
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* A name that resolves to nothing is usually a typo or a half-remembered nesting, not a
|
|
70
|
+
* different concept, so the ranked search comes first and prefix overlap is the fallback
|
|
71
|
+
* that survives a wrong tail: `Hds::Buton` still reaches `Hds::Button`.
|
|
72
|
+
*/
|
|
73
|
+
const suggestComponentNames = (componentName, limit) => {
|
|
74
|
+
const { hits } = searchComponents({ query: componentName, limit });
|
|
75
|
+
if (hits.length > 0) {
|
|
76
|
+
return hits.map((hit) => hit.name);
|
|
77
|
+
}
|
|
78
|
+
const requested = normalizeComponentName(componentName);
|
|
79
|
+
return componentRecords
|
|
80
|
+
.map((component) => ({
|
|
81
|
+
name: component.name,
|
|
82
|
+
overlap: getCommonPrefixLength(normalizeComponentName(component.name), requested),
|
|
83
|
+
}))
|
|
84
|
+
.filter((candidate) => candidate.overlap > 0)
|
|
85
|
+
.sort((left, right) => right.overlap === left.overlap
|
|
86
|
+
? left.name.localeCompare(right.name)
|
|
87
|
+
: right.overlap - left.overlap)
|
|
88
|
+
.slice(0, limit)
|
|
89
|
+
.map((candidate) => candidate.name);
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
getMeta: () => ({
|
|
93
|
+
totalComponentCount: componentRecords.length,
|
|
94
|
+
source,
|
|
95
|
+
}),
|
|
96
|
+
listComponents: () => componentRecords.map((component) => toComponentSummary(component)),
|
|
97
|
+
getComponentByName: (componentName) => {
|
|
98
|
+
return componentLookup.get(normalizeLookupValue(componentName)) ?? null;
|
|
99
|
+
},
|
|
100
|
+
searchComponents,
|
|
101
|
+
suggestComponentNames,
|
|
102
|
+
};
|
|
103
|
+
};
|
|
104
|
+
const componentCatalogLoader = createCatalogLoader({
|
|
105
|
+
specifier: "@hashicorp/design-system-components/component-catalog.json",
|
|
106
|
+
anchors: ["project-root", "default"],
|
|
107
|
+
create: (value, source) => createComponentCatalogStore(parseComponentCatalog(value), source),
|
|
108
|
+
});
|
|
109
|
+
export const loadComponentCatalog = () => {
|
|
110
|
+
return componentCatalogLoader.load();
|
|
111
|
+
};
|
|
112
|
+
export const getOrLoadComponentStore = () => {
|
|
113
|
+
return componentCatalogLoader.getOrLoad();
|
|
114
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright IBM Corp. 2021, 2026
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
*/
|
|
5
|
+
import { normalizeLookupValue } from "../../catalog/normalize.js";
|
|
6
|
+
const HDS_NAMESPACE_PREFIX = "hds::";
|
|
7
|
+
export const normalizeComponentName = (value) => {
|
|
8
|
+
const normalized = normalizeLookupValue(value);
|
|
9
|
+
return normalized.startsWith(HDS_NAMESPACE_PREFIX)
|
|
10
|
+
? normalized.slice(HDS_NAMESPACE_PREFIX.length)
|
|
11
|
+
: normalized;
|
|
12
|
+
};
|
|
13
|
+
export const getComponentLookupKeys = (entry) => {
|
|
14
|
+
const invocation = normalizeLookupValue(entry.name);
|
|
15
|
+
const withoutNamespace = normalizeComponentName(entry.name);
|
|
16
|
+
return [
|
|
17
|
+
invocation,
|
|
18
|
+
withoutNamespace,
|
|
19
|
+
invocation.replaceAll("::", ""),
|
|
20
|
+
withoutNamespace.replaceAll("::", ""),
|
|
21
|
+
// the specifier an import uses, which is the one form a caller can copy out of a
|
|
22
|
+
// stack trace or an Embroider error rather than retype
|
|
23
|
+
normalizeLookupValue(entry.modulePath),
|
|
24
|
+
];
|
|
25
|
+
};
|
|
26
|
+
const resolveArgValues = (arg, valueSets) => {
|
|
27
|
+
if (arg.valuesRef === undefined) {
|
|
28
|
+
return arg;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
const values = valueSets[arg.valuesRef];
|
|
32
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
33
|
+
const { valuesRef: _valuesRef, ...rest } = arg;
|
|
34
|
+
return values === undefined ? rest : { ...rest, values };
|
|
35
|
+
}
|
|
36
|
+
};
|
|
37
|
+
export const toComponentRecord = (entry, valueSets = {}) => {
|
|
38
|
+
return {
|
|
39
|
+
name: entry.name,
|
|
40
|
+
modulePath: entry.modulePath,
|
|
41
|
+
...(entry.docsPath === undefined ? {} : { docsPath: entry.docsPath }),
|
|
42
|
+
...(entry.element === undefined ? {} : { element: entry.element }),
|
|
43
|
+
args: entry.args.map((arg) => resolveArgValues(arg, valueSets)),
|
|
44
|
+
blocks: entry.blocks,
|
|
45
|
+
};
|
|
46
|
+
};
|
|
47
|
+
export const toComponentSummary = (component) => {
|
|
48
|
+
return {
|
|
49
|
+
name: component.name,
|
|
50
|
+
modulePath: component.modulePath,
|
|
51
|
+
...(component.docsPath === undefined
|
|
52
|
+
? {}
|
|
53
|
+
: { docsPath: component.docsPath }),
|
|
54
|
+
};
|
|
55
|
+
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright IBM Corp. 2021, 2026
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
*/
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const componentArgSchema = z.object({
|
|
7
|
+
name: z.string().min(1),
|
|
8
|
+
type: z.string().min(1),
|
|
9
|
+
required: z.boolean(),
|
|
10
|
+
values: z.array(z.string()).optional(),
|
|
11
|
+
valuesRef: z.string().min(1).optional(),
|
|
12
|
+
inheritedFrom: z.string().min(1).optional(),
|
|
13
|
+
});
|
|
14
|
+
const componentBlockYieldSchema = z.object({
|
|
15
|
+
name: z.string().min(1),
|
|
16
|
+
type: z.string().min(1),
|
|
17
|
+
});
|
|
18
|
+
const componentBlockSchema = z.object({
|
|
19
|
+
name: z.string().min(1),
|
|
20
|
+
yields: z.array(componentBlockYieldSchema),
|
|
21
|
+
});
|
|
22
|
+
export const componentCatalogEntrySchema = z
|
|
23
|
+
.object({
|
|
24
|
+
name: z.string().min(1),
|
|
25
|
+
modulePath: z.string().min(1),
|
|
26
|
+
docsPath: z.string().min(1).optional(),
|
|
27
|
+
element: z.string().min(1).optional(),
|
|
28
|
+
args: z.array(componentArgSchema),
|
|
29
|
+
blocks: z.array(componentBlockSchema),
|
|
30
|
+
})
|
|
31
|
+
.catchall(z.any());
|
|
32
|
+
export const componentCatalogSchema = z
|
|
33
|
+
.object({
|
|
34
|
+
valueSets: z.record(z.string(), z.array(z.string())).optional(),
|
|
35
|
+
components: z.array(componentCatalogEntrySchema),
|
|
36
|
+
})
|
|
37
|
+
.catchall(z.any());
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright IBM Corp. 2021, 2026
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
*/
|
|
5
|
+
export const SITE_BASE_URL = "https://helios.hashicorp.design/";
|
|
6
|
+
export const DOCS_CATALOG_FILE_NAME = "docs-catalog.json";
|
|
7
|
+
export const NOT_FOUND_SUGGESTION_LIMIT = 5;
|
|
8
|
+
export const SNIPPET_MAX_LENGTH = 300;
|
|
9
|
+
// BM25 ranking tunables
|
|
10
|
+
// https://lucene.apache.org/core/9_12_3/core/org/apache/lucene/search/similarities/BM25Similarity.html
|
|
11
|
+
export const BM25_K1 = 1.6;
|
|
12
|
+
export const BM25_B = 0.6;
|
|
13
|
+
// a page match says "this page is about your query", a chunk match says "this passage is"
|
|
14
|
+
export const PAGE_SCORE_WEIGHT = 1.5;
|
|
15
|
+
export const CHUNK_SCORE_WEIGHT = 1;
|
|
16
|
+
export const PAGE_NAME_MATCH_WEIGHT = 1;
|
|
17
|
+
// how many passages any one page may contribute to a result window
|
|
18
|
+
export const MAX_HITS_PER_ROUTE = 3;
|
|
19
|
+
export const PAGE_FIELD_WEIGHTS = {
|
|
20
|
+
name: 3,
|
|
21
|
+
title: 3,
|
|
22
|
+
keywords: 5,
|
|
23
|
+
description: 1,
|
|
24
|
+
};
|
|
25
|
+
export const CHUNK_FIELD_WEIGHTS = {
|
|
26
|
+
heading: 6,
|
|
27
|
+
ancestorHeadings: 2,
|
|
28
|
+
tab: 1,
|
|
29
|
+
body: 1,
|
|
30
|
+
};
|
|
31
|
+
export const VERSION_HISTORY_TAB = "Version history";
|
|
32
|
+
export const VERSION_HISTORY_SCORE_MULTIPLIER = 0.6;
|
|
33
|
+
// reduce results for partial matches
|
|
34
|
+
export const PAGE_ONLY_CHUNK_LIMIT = 3;
|
|
35
|
+
export const STOPWORDS = new Set([
|
|
36
|
+
"about",
|
|
37
|
+
"above",
|
|
38
|
+
"after",
|
|
39
|
+
"again",
|
|
40
|
+
"against",
|
|
41
|
+
"all",
|
|
42
|
+
"also",
|
|
43
|
+
"am",
|
|
44
|
+
"an",
|
|
45
|
+
"and",
|
|
46
|
+
"any",
|
|
47
|
+
"are",
|
|
48
|
+
"as",
|
|
49
|
+
"at",
|
|
50
|
+
"be",
|
|
51
|
+
"because",
|
|
52
|
+
"been",
|
|
53
|
+
"before",
|
|
54
|
+
"being",
|
|
55
|
+
"below",
|
|
56
|
+
"between",
|
|
57
|
+
"both",
|
|
58
|
+
"but",
|
|
59
|
+
"by",
|
|
60
|
+
"can",
|
|
61
|
+
"cannot",
|
|
62
|
+
"could",
|
|
63
|
+
"did",
|
|
64
|
+
"do",
|
|
65
|
+
"does",
|
|
66
|
+
"doing",
|
|
67
|
+
"down",
|
|
68
|
+
"during",
|
|
69
|
+
"each",
|
|
70
|
+
"few",
|
|
71
|
+
"for",
|
|
72
|
+
"from",
|
|
73
|
+
"further",
|
|
74
|
+
"had",
|
|
75
|
+
"has",
|
|
76
|
+
"have",
|
|
77
|
+
"having",
|
|
78
|
+
"he",
|
|
79
|
+
"her",
|
|
80
|
+
"here",
|
|
81
|
+
"hers",
|
|
82
|
+
"herself",
|
|
83
|
+
"him",
|
|
84
|
+
"himself",
|
|
85
|
+
"his",
|
|
86
|
+
"how",
|
|
87
|
+
"if",
|
|
88
|
+
"in",
|
|
89
|
+
"into",
|
|
90
|
+
"is",
|
|
91
|
+
"it",
|
|
92
|
+
"its",
|
|
93
|
+
"itself",
|
|
94
|
+
"just",
|
|
95
|
+
"me",
|
|
96
|
+
"more",
|
|
97
|
+
"most",
|
|
98
|
+
"my",
|
|
99
|
+
"myself",
|
|
100
|
+
"no",
|
|
101
|
+
"nor",
|
|
102
|
+
"not",
|
|
103
|
+
"now",
|
|
104
|
+
"of",
|
|
105
|
+
"off",
|
|
106
|
+
"on",
|
|
107
|
+
"once",
|
|
108
|
+
"only",
|
|
109
|
+
"or",
|
|
110
|
+
"other",
|
|
111
|
+
"ought",
|
|
112
|
+
"our",
|
|
113
|
+
"ours",
|
|
114
|
+
"ourselves",
|
|
115
|
+
"out",
|
|
116
|
+
"over",
|
|
117
|
+
"own",
|
|
118
|
+
"same",
|
|
119
|
+
"she",
|
|
120
|
+
"should",
|
|
121
|
+
"so",
|
|
122
|
+
"some",
|
|
123
|
+
"such",
|
|
124
|
+
"than",
|
|
125
|
+
"that",
|
|
126
|
+
"the",
|
|
127
|
+
"their",
|
|
128
|
+
"theirs",
|
|
129
|
+
"them",
|
|
130
|
+
"themselves",
|
|
131
|
+
"then",
|
|
132
|
+
"there",
|
|
133
|
+
"these",
|
|
134
|
+
"they",
|
|
135
|
+
"this",
|
|
136
|
+
"those",
|
|
137
|
+
"through",
|
|
138
|
+
"to",
|
|
139
|
+
"too",
|
|
140
|
+
"under",
|
|
141
|
+
"until",
|
|
142
|
+
"up",
|
|
143
|
+
"very",
|
|
144
|
+
"was",
|
|
145
|
+
"we",
|
|
146
|
+
"were",
|
|
147
|
+
"what",
|
|
148
|
+
"when",
|
|
149
|
+
"where",
|
|
150
|
+
"which",
|
|
151
|
+
"while",
|
|
152
|
+
"who",
|
|
153
|
+
"whom",
|
|
154
|
+
"why",
|
|
155
|
+
"will",
|
|
156
|
+
"with",
|
|
157
|
+
"would",
|
|
158
|
+
"you",
|
|
159
|
+
"your",
|
|
160
|
+
"yours",
|
|
161
|
+
"yourself",
|
|
162
|
+
"yourselves",
|
|
163
|
+
]);
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright IBM Corp. 2021, 2026
|
|
3
|
+
* SPDX-License-Identifier: MPL-2.0
|
|
4
|
+
*/
|
|
5
|
+
import { readFileSync } from "node:fs";
|
|
6
|
+
import { join } from "node:path";
|
|
7
|
+
import { memoizeCatalogLoader } from "../../catalog/loader.js";
|
|
8
|
+
import { getMcpPackageDirectory } from "../../catalog/project-root.js";
|
|
9
|
+
import { normalizeLookupValue } from "../../catalog/normalize.js";
|
|
10
|
+
import { CHUNK_FIELD_WEIGHTS, CHUNK_SCORE_WEIGHT, DOCS_CATALOG_FILE_NAME, NOT_FOUND_SUGGESTION_LIMIT, MAX_HITS_PER_ROUTE, PAGE_FIELD_WEIGHTS, PAGE_NAME_MATCH_WEIGHT, PAGE_ONLY_CHUNK_LIMIT, PAGE_SCORE_WEIGHT, VERSION_HISTORY_SCORE_MULTIPLIER, VERSION_HISTORY_TAB, } from "./constants.js";
|
|
11
|
+
import { buildSnippet, getAncestorHeadings, getChunkHeading, getPageNameTerms, matchesDocsPath, matchesFilters, normalizeRoute, toChunkRecord, toPageRecord, } from "./lookup.js";
|
|
12
|
+
import { docsCatalogSchema } from "./schema.js";
|
|
13
|
+
import { buildBm25Index, getNameMatchRatio, hasTerm, scoreBm25, } from "./search-index.js";
|
|
14
|
+
import { tokenize, tokenizeQuery } from "./tokenize.js";
|
|
15
|
+
import { getCommonPrefixLength } from "../rank.js";
|
|
16
|
+
// splits the raw query into the words a caller would recognize
|
|
17
|
+
const QUERY_WORD_PATTERN = /[A-Za-z0-9][A-Za-z0-9.:@_-]*/g;
|
|
18
|
+
const roundScore = (value) => {
|
|
19
|
+
return Math.round(value * 10_000) / 10_000;
|
|
20
|
+
};
|
|
21
|
+
export const parseDocsCatalog = (value) => {
|
|
22
|
+
return docsCatalogSchema.parse(value);
|
|
23
|
+
};
|
|
24
|
+
// trims the result window so no single page can hold all of it, in score order
|
|
25
|
+
const diversifyByPage = (candidates, limit) => {
|
|
26
|
+
const kept = [];
|
|
27
|
+
const overflow = [];
|
|
28
|
+
const countsByPage = new Map();
|
|
29
|
+
for (const candidate of candidates) {
|
|
30
|
+
if (kept.length >= limit) {
|
|
31
|
+
break;
|
|
32
|
+
}
|
|
33
|
+
const seen = countsByPage.get(candidate.pageIndex) ?? 0;
|
|
34
|
+
if (seen >= MAX_HITS_PER_ROUTE) {
|
|
35
|
+
overflow.push(candidate);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
countsByPage.set(candidate.pageIndex, seen + 1);
|
|
39
|
+
kept.push(candidate);
|
|
40
|
+
}
|
|
41
|
+
return kept.length >= limit
|
|
42
|
+
? kept
|
|
43
|
+
: [...kept, ...overflow.slice(0, limit - kept.length)];
|
|
44
|
+
};
|
|
45
|
+
export const createDocsCatalogStore = (catalog) => {
|
|
46
|
+
const pages = [];
|
|
47
|
+
const pageNameTerms = [];
|
|
48
|
+
const chunks = [];
|
|
49
|
+
const chunkPageIndexes = [];
|
|
50
|
+
const pageIndexByRoute = new Map();
|
|
51
|
+
const chunkIndexById = new Map();
|
|
52
|
+
const sections = new Set();
|
|
53
|
+
const tabs = new Set();
|
|
54
|
+
for (const page of Object.values(catalog.pages)) {
|
|
55
|
+
const pageIndex = pages.length;
|
|
56
|
+
const record = toPageRecord(page);
|
|
57
|
+
pages.push(record);
|
|
58
|
+
pageNameTerms.push(getPageNameTerms(record));
|
|
59
|
+
pageIndexByRoute.set(normalizeRoute(page.route), pageIndex);
|
|
60
|
+
sections.add(page.section);
|
|
61
|
+
for (const chunk of page.chunks) {
|
|
62
|
+
chunkIndexById.set(chunk.id.toLowerCase(), chunks.length);
|
|
63
|
+
chunkPageIndexes.push(pageIndex);
|
|
64
|
+
chunks.push(toChunkRecord(chunk, page));
|
|
65
|
+
if (chunk.tab !== undefined) {
|
|
66
|
+
tabs.add(chunk.tab);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const normalizedSections = new Set([...sections].map((section) => normalizeLookupValue(section)));
|
|
71
|
+
const normalizedTabs = new Set([...tabs].map((tab) => normalizeLookupValue(tab)));
|
|
72
|
+
let indexes = null;
|
|
73
|
+
const getIndexes = () => {
|
|
74
|
+
if (indexes !== null) {
|
|
75
|
+
return indexes;
|
|
76
|
+
}
|
|
77
|
+
const pageDocuments = pages.map((page) => [
|
|
78
|
+
{ value: page.route, weight: PAGE_FIELD_WEIGHTS.name },
|
|
79
|
+
{ value: page.title, weight: PAGE_FIELD_WEIGHTS.title },
|
|
80
|
+
{ value: page.keywords, weight: PAGE_FIELD_WEIGHTS.keywords },
|
|
81
|
+
{
|
|
82
|
+
value: [page.description, page.caption].filter((value) => value !== undefined),
|
|
83
|
+
weight: PAGE_FIELD_WEIGHTS.description,
|
|
84
|
+
},
|
|
85
|
+
]);
|
|
86
|
+
const chunkDocuments = chunks.map((chunk) => [
|
|
87
|
+
{ value: getChunkHeading(chunk), weight: CHUNK_FIELD_WEIGHTS.heading },
|
|
88
|
+
{
|
|
89
|
+
value: getAncestorHeadings(chunk),
|
|
90
|
+
weight: CHUNK_FIELD_WEIGHTS.ancestorHeadings,
|
|
91
|
+
},
|
|
92
|
+
{ value: chunk.tab, weight: CHUNK_FIELD_WEIGHTS.tab },
|
|
93
|
+
{ value: chunk.text, weight: CHUNK_FIELD_WEIGHTS.body },
|
|
94
|
+
{ value: chunk.symbols, weight: CHUNK_FIELD_WEIGHTS.body },
|
|
95
|
+
]);
|
|
96
|
+
indexes = {
|
|
97
|
+
page: buildBm25Index(pageDocuments),
|
|
98
|
+
chunk: buildBm25Index(chunkDocuments),
|
|
99
|
+
};
|
|
100
|
+
return indexes;
|
|
101
|
+
};
|
|
102
|
+
const analyzeQuery = (query, pageIndex, chunkIndex) => {
|
|
103
|
+
const displayByTerm = new Map();
|
|
104
|
+
const matchedTerms = [];
|
|
105
|
+
const unmatchedTerms = [];
|
|
106
|
+
for (const word of query.match(QUERY_WORD_PATTERN) ?? []) {
|
|
107
|
+
const wordTerms = tokenize(word);
|
|
108
|
+
// a word that is entirely stopwords is neither a match nor a miss
|
|
109
|
+
if (wordTerms.length === 0) {
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
let isMatched = false;
|
|
113
|
+
for (const term of wordTerms) {
|
|
114
|
+
if (!displayByTerm.has(term)) {
|
|
115
|
+
displayByTerm.set(term, word);
|
|
116
|
+
}
|
|
117
|
+
if (hasTerm(pageIndex, term) || hasTerm(chunkIndex, term)) {
|
|
118
|
+
isMatched = true;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const bucket = isMatched ? matchedTerms : unmatchedTerms;
|
|
122
|
+
if (!bucket.includes(word)) {
|
|
123
|
+
bucket.push(word);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
const terms = tokenizeQuery(query);
|
|
127
|
+
return {
|
|
128
|
+
terms,
|
|
129
|
+
termSet: new Set(terms),
|
|
130
|
+
displayByTerm,
|
|
131
|
+
matchedTerms,
|
|
132
|
+
unmatchedTerms,
|
|
133
|
+
};
|
|
134
|
+
};
|
|
135
|
+
const resolveFilters = (query) => {
|
|
136
|
+
const filters = {};
|
|
137
|
+
const unknownFilters = [];
|
|
138
|
+
/**
|
|
139
|
+
* A value that normalizes to nothing — `" "`, or a `docsPath` of `"/"` — is reported and
|
|
140
|
+
* then dropped. Applying it would match no chunk at all, and echoing it back in `filters`
|
|
141
|
+
* would name an empty string as the filter that did the excluding.
|
|
142
|
+
*/
|
|
143
|
+
if (query.section !== undefined) {
|
|
144
|
+
const section = normalizeLookupValue(query.section);
|
|
145
|
+
if (section.length > 0) {
|
|
146
|
+
filters.section = section;
|
|
147
|
+
}
|
|
148
|
+
if (section.length === 0 || !normalizedSections.has(section)) {
|
|
149
|
+
unknownFilters.push(`section: ${query.section}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
if (query.tab !== undefined) {
|
|
153
|
+
const tab = normalizeLookupValue(query.tab);
|
|
154
|
+
if (tab.length > 0) {
|
|
155
|
+
filters.tab = tab;
|
|
156
|
+
}
|
|
157
|
+
if (tab.length === 0 || !normalizedTabs.has(tab)) {
|
|
158
|
+
unknownFilters.push(`tab: ${query.tab}`);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (query.docsPath !== undefined) {
|
|
162
|
+
const docsPath = normalizeRoute(query.docsPath);
|
|
163
|
+
if (docsPath.length > 0) {
|
|
164
|
+
filters.docsPath = docsPath;
|
|
165
|
+
}
|
|
166
|
+
const isKnownRoute = docsPath.length > 0 &&
|
|
167
|
+
[...pageIndexByRoute.keys()].some((route) => matchesDocsPath(route, docsPath));
|
|
168
|
+
if (!isKnownRoute) {
|
|
169
|
+
unknownFilters.push(`docsPath: ${query.docsPath}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
return { filters, unknownFilters };
|
|
173
|
+
};
|
|
174
|
+
const search = (query) => {
|
|
175
|
+
const { page: pageIndex, chunk: chunkIndex } = getIndexes();
|
|
176
|
+
const analysis = analyzeQuery(query.query, pageIndex, chunkIndex);
|
|
177
|
+
const { filters, unknownFilters } = resolveFilters(query);
|
|
178
|
+
const emptyOutcome = {
|
|
179
|
+
filters,
|
|
180
|
+
unknownFilters,
|
|
181
|
+
matchedTerms: analysis.matchedTerms,
|
|
182
|
+
unmatchedTerms: analysis.unmatchedTerms,
|
|
183
|
+
totalMatches: 0,
|
|
184
|
+
hits: [],
|
|
185
|
+
};
|
|
186
|
+
if (analysis.terms.length === 0) {
|
|
187
|
+
return emptyOutcome;
|
|
188
|
+
}
|
|
189
|
+
const pageScores = scoreBm25(pageIndex, analysis.terms);
|
|
190
|
+
const chunkScores = scoreBm25(chunkIndex, analysis.terms);
|
|
191
|
+
const nameMatchRatios = pageNameTerms.map((nameTerms) => getNameMatchRatio(analysis.termSet, nameTerms));
|
|
192
|
+
const isVersionHistoryScoped = filters.tab === normalizeLookupValue(VERSION_HISTORY_TAB);
|
|
193
|
+
const candidates = [];
|
|
194
|
+
const pageOnlyCounts = new Map();
|
|
195
|
+
for (const [index, chunk] of chunks.entries()) {
|
|
196
|
+
const chunkScore = chunkScores.scores[index];
|
|
197
|
+
const ownerPageIndex = chunkPageIndexes[index];
|
|
198
|
+
const pageScore = pageScores.scores[ownerPageIndex];
|
|
199
|
+
if (chunkScore === 0 && pageScore === 0) {
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
if (!matchesFilters(chunk, filters)) {
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
if (chunkScore === 0) {
|
|
206
|
+
const seen = pageOnlyCounts.get(ownerPageIndex) ?? 0;
|
|
207
|
+
if (seen >= PAGE_ONLY_CHUNK_LIMIT) {
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
pageOnlyCounts.set(ownerPageIndex, seen + 1);
|
|
211
|
+
}
|
|
212
|
+
let score = PAGE_SCORE_WEIGHT * pageScore + CHUNK_SCORE_WEIGHT * chunkScore;
|
|
213
|
+
score *= 1 + PAGE_NAME_MATCH_WEIGHT * nameMatchRatios[ownerPageIndex];
|
|
214
|
+
if (chunk.tab === VERSION_HISTORY_TAB && !isVersionHistoryScoped) {
|
|
215
|
+
score *= VERSION_HISTORY_SCORE_MULTIPLIER;
|
|
216
|
+
}
|
|
217
|
+
candidates.push({ chunkIndex: index, pageIndex: ownerPageIndex, score });
|
|
218
|
+
}
|
|
219
|
+
if (candidates.length === 0) {
|
|
220
|
+
return emptyOutcome;
|
|
221
|
+
}
|
|
222
|
+
candidates.sort((left, right) => right.score === left.score
|
|
223
|
+
? left.chunkIndex - right.chunkIndex
|
|
224
|
+
: right.score - left.score);
|
|
225
|
+
const topScore = candidates[0].score;
|
|
226
|
+
const hits = diversifyByPage(candidates, query.limit).map((candidate) => {
|
|
227
|
+
const chunk = chunks[candidate.chunkIndex];
|
|
228
|
+
const pageMatched = pageScores.matchedTermsByDocument.get(candidate.pageIndex) ?? new Set();
|
|
229
|
+
const pageMatchedTerms = [...pageMatched]
|
|
230
|
+
.map((term) => analysis.displayByTerm.get(term) ?? term)
|
|
231
|
+
.filter((term, index, all) => all.indexOf(term) === index);
|
|
232
|
+
return {
|
|
233
|
+
chunk,
|
|
234
|
+
page: pages[candidate.pageIndex],
|
|
235
|
+
score: roundScore(candidate.score),
|
|
236
|
+
relScore: topScore === 0 ? 0 : roundScore(candidate.score / topScore),
|
|
237
|
+
pageMatchedTerms,
|
|
238
|
+
snippet: buildSnippet(chunk.text, analysis.termSet),
|
|
239
|
+
};
|
|
240
|
+
});
|
|
241
|
+
return {
|
|
242
|
+
filters,
|
|
243
|
+
unknownFilters,
|
|
244
|
+
matchedTerms: analysis.matchedTerms,
|
|
245
|
+
unmatchedTerms: analysis.unmatchedTerms,
|
|
246
|
+
totalMatches: candidates.length,
|
|
247
|
+
hits,
|
|
248
|
+
};
|
|
249
|
+
};
|
|
250
|
+
// a chunk id is route-shaped, so it tolerates the same stray casing and slashes a route does
|
|
251
|
+
const getChunkById = (id) => {
|
|
252
|
+
const index = chunkIndexById.get(normalizeRoute(id));
|
|
253
|
+
return index === undefined ? null : chunks[index];
|
|
254
|
+
};
|
|
255
|
+
const getChildChunks = (id) => {
|
|
256
|
+
const parent = getChunkById(id);
|
|
257
|
+
if (parent === null) {
|
|
258
|
+
return [];
|
|
259
|
+
}
|
|
260
|
+
// scoped to the parent's own page, so a route that prefixes another cannot claim its chunks
|
|
261
|
+
return chunks.filter((chunk) => chunk.route === parent.route &&
|
|
262
|
+
(chunk.id.startsWith(`${parent.id}/`) ||
|
|
263
|
+
chunk.id.startsWith(`${parent.id}#`)));
|
|
264
|
+
};
|
|
265
|
+
const suggestChunkIds = (id) => {
|
|
266
|
+
const requested = normalizeRoute(id);
|
|
267
|
+
const route = requested.split("#")[0];
|
|
268
|
+
const pageIndex = pageIndexByRoute.get(route);
|
|
269
|
+
if (pageIndex !== undefined) {
|
|
270
|
+
return chunks
|
|
271
|
+
.filter((chunk) => chunk.route === pages[pageIndex].route)
|
|
272
|
+
.slice(0, NOT_FOUND_SUGGESTION_LIMIT)
|
|
273
|
+
.map((chunk) => chunk.id);
|
|
274
|
+
}
|
|
275
|
+
return (chunks
|
|
276
|
+
.map((chunk) => ({
|
|
277
|
+
id: chunk.id,
|
|
278
|
+
overlap: getCommonPrefixLength(chunk.id.toLowerCase(), requested),
|
|
279
|
+
}))
|
|
280
|
+
.filter((candidate) => candidate.overlap > 0)
|
|
281
|
+
// codepoint order, never locale order: these keys carry `/`, `#` and `-`
|
|
282
|
+
.sort((left, right) => {
|
|
283
|
+
if (right.overlap !== left.overlap) {
|
|
284
|
+
return right.overlap - left.overlap;
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
return left.id < right.id ? -1 : left.id > right.id ? 1 : 0;
|
|
288
|
+
}
|
|
289
|
+
})
|
|
290
|
+
.slice(0, NOT_FOUND_SUGGESTION_LIMIT)
|
|
291
|
+
.map((candidate) => candidate.id));
|
|
292
|
+
};
|
|
293
|
+
return {
|
|
294
|
+
getMeta: () => ({
|
|
295
|
+
bundledAt: catalog.bundledAt,
|
|
296
|
+
siteBaseUrl: catalog.siteBaseUrl,
|
|
297
|
+
totalPageCount: pages.length,
|
|
298
|
+
totalChunkCount: chunks.length,
|
|
299
|
+
}),
|
|
300
|
+
search,
|
|
301
|
+
getChunkById,
|
|
302
|
+
getPageByRoute: (route) => {
|
|
303
|
+
const index = pageIndexByRoute.get(normalizeRoute(route));
|
|
304
|
+
return index === undefined ? null : pages[index];
|
|
305
|
+
},
|
|
306
|
+
getChildChunks,
|
|
307
|
+
suggestChunkIds,
|
|
308
|
+
listSections: () => [...sections].sort(),
|
|
309
|
+
listTabs: () => [...tabs].sort(),
|
|
310
|
+
};
|
|
311
|
+
};
|
|
312
|
+
const docsCatalogLoader = memoizeCatalogLoader(() => {
|
|
313
|
+
const catalogPath = join(getMcpPackageDirectory(), DOCS_CATALOG_FILE_NAME);
|
|
314
|
+
const rawCatalog = readFileSync(catalogPath, "utf8");
|
|
315
|
+
return createDocsCatalogStore(parseDocsCatalog(JSON.parse(rawCatalog)));
|
|
316
|
+
});
|
|
317
|
+
export const loadDocsCatalog = () => {
|
|
318
|
+
return docsCatalogLoader.load();
|
|
319
|
+
};
|
|
320
|
+
export const getOrLoadDocsStore = () => {
|
|
321
|
+
return docsCatalogLoader.getOrLoad();
|
|
322
|
+
};
|