@alauda/doom 2.8.1 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/cli/load-config.js
CHANGED
|
@@ -14,8 +14,8 @@ import { transformerMetaHighlight, transformerMetaWordHighlight, transformerNota
|
|
|
14
14
|
import { difference } from 'es-toolkit';
|
|
15
15
|
import { glob } from 'tinyglobby';
|
|
16
16
|
import { cyan } from 'yoctocolors';
|
|
17
|
-
import { apiPlugin, autoSidebarPlugin, autoTocPlugin, createTransformerCallouts, directivesPlugin, globalPlugin, mermaidPlugin, permissionPlugin, replacePlugin, } from "../plugins/index.js";
|
|
18
|
-
import { isExplicitlyUnversioned, normalizeSlash, UNVERSIONED, } from "../shared/index.js";
|
|
17
|
+
import { apiPlugin, autoSidebarPlugin, autoTocPlugin, createTransformerCallouts, directivesPlugin, globalPlugin, mermaidPlugin, permissionPlugin, replacePlugin, searchIndexPlugin, } from "../plugins/index.js";
|
|
18
|
+
import { isExplicitlyUnversioned, Language, normalizeSlash, SUPPORTED_LANGUAGES, UNVERSIONED, } from "../shared/index.js";
|
|
19
19
|
import { pathExists, pkgResolve, resolveStaticConfig } from "../utils/index.js";
|
|
20
20
|
import { DEFAULT_CONFIG_NAME, DEFAULT_EXTENSIONS, I18N_FILE, SITES_FILE, YAML_EXTENSIONS, } from "./constants.js";
|
|
21
21
|
import { defaultGitHubUrl, isDoc } from "./helpers.js";
|
|
@@ -94,6 +94,16 @@ const getCommonConfig = async ({ config, root, configFilePath, base, version, do
|
|
|
94
94
|
continue;
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
|
+
// Only a language doom supports is a language. Any other directory, a
|
|
98
|
+
// `scripts/` or `images/` say, used to become one and get a
|
|
99
|
+
// `/<name>/products` page with no words to render, which failed the
|
|
100
|
+
// build. It is left out of the site like `shared/` instead: it stays in
|
|
101
|
+
// `allLanguages`, so it ends up in `unusedLanguages` and its routes are
|
|
102
|
+
// excluded below.
|
|
103
|
+
if (!Object.hasOwn(Language, name)) {
|
|
104
|
+
logger.warn(`Skipped \`${cyan(path.relative(process.cwd(), path.join(root, name)))}\`: not a language directory (doom builds ${SUPPORTED_LANGUAGES.join(', ')}), so no pages are built from it, like \`shared\``);
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
97
107
|
locales.push({
|
|
98
108
|
lang: name,
|
|
99
109
|
label: name,
|
|
@@ -190,6 +200,7 @@ const getCommonConfig = async ({ config, root, configFilePath, base, version, do
|
|
|
190
200
|
localBasePath,
|
|
191
201
|
force,
|
|
192
202
|
}),
|
|
203
|
+
searchIndexPlugin(),
|
|
193
204
|
].filter(Boolean),
|
|
194
205
|
search: algoliaOptions ? false : undefined,
|
|
195
206
|
builderConfig: {
|
package/lib/plugins/index.d.ts
CHANGED
package/lib/plugins/index.js
CHANGED
|
@@ -42,6 +42,14 @@ const issuesToListItems = (issues, lang, isMdx) => issues
|
|
|
42
42
|
})
|
|
43
43
|
.filter(Boolean);
|
|
44
44
|
const { JIRA_USERNAME, JIRA_PASSWORD } = process.env;
|
|
45
|
+
const DEFAULT_JIRA_BASE_URL = 'https://jira.alauda.cn';
|
|
46
|
+
/**
|
|
47
|
+
* Where Jira is, without the `/rest/...` part. Unlike the credentials above,
|
|
48
|
+
* which are read once when this module loads, it is read for each request.
|
|
49
|
+
* Empty counts as unset — an unset variable often arrives that way — and a
|
|
50
|
+
* trailing `/` is dropped so the path appended to it is not doubled.
|
|
51
|
+
*/
|
|
52
|
+
const jiraBaseUrl = () => (process.env.JIRA_BASE_URL || DEFAULT_JIRA_BASE_URL).replace(/\/+$/, '');
|
|
45
53
|
let warned = false;
|
|
46
54
|
const NO_ISSUE_MAPPER = {
|
|
47
55
|
zh: '此次发版无相关问题。',
|
|
@@ -87,7 +95,7 @@ const resolveRelease_ = async (releaseTemplates, releaseQuery, isMdx) => {
|
|
|
87
95
|
let issues;
|
|
88
96
|
try {
|
|
89
97
|
;
|
|
90
|
-
({ issues } = await xfetch(
|
|
98
|
+
({ issues } = await xfetch(`${jiraBaseUrl()}/rest/api/2/search?${new URLSearchParams({ jql })}`, { headers: { Authorization } }));
|
|
91
99
|
}
|
|
92
100
|
catch (err) {
|
|
93
101
|
if (err instanceof ResponseError) {
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { RspressPlugin } from '@rspress/core';
|
|
2
|
+
export interface SearchIndexPluginOptions {
|
|
3
|
+
/** @default 64 */
|
|
4
|
+
maxTokenLength?: number;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Keeps pathologically long tokens out of the browser-side search index.
|
|
8
|
+
*
|
|
9
|
+
* Without this, three pages of the connectors documentation — each embedding a
|
|
10
|
+
* base64-encoded SVG icon inside a YAML sample — turned a 1.1 MB index file
|
|
11
|
+
* into ~2 GB of browser memory and crashed the tab with `Out of Memory` as soon
|
|
12
|
+
* as the user opened the search box. Stripping ~20,000 characters (2% of the
|
|
13
|
+
* indexed text) brought that down to 72 MB.
|
|
14
|
+
*/
|
|
15
|
+
export declare const searchIndexPlugin: ({ maxTokenLength, }?: SearchIndexPluginOptions) => RspressPlugin;
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Longest token kept in the search index, in characters.
|
|
3
|
+
*
|
|
4
|
+
* The browser-side search engine (FlexSearch) is configured by rspress with
|
|
5
|
+
* `tokenize: 'full'`, which indexes *every substring* of every token. The cost
|
|
6
|
+
* of a token therefore grows with the square of its length: a 12,000-character
|
|
7
|
+
* token expands into ~70 million substrings and exhausts the tab's memory
|
|
8
|
+
* before the index is finished.
|
|
9
|
+
*
|
|
10
|
+
* Real prose never reaches this length — the longest English words run about
|
|
11
|
+
* 20-30 characters and long code identifiers about 40-50. Tokens above the
|
|
12
|
+
* limit are invariably machine data that nobody searches for: base64 payloads,
|
|
13
|
+
* hashes, JWTs.
|
|
14
|
+
*/
|
|
15
|
+
const DEFAULT_MAX_TOKEN_LENGTH = 64;
|
|
16
|
+
/**
|
|
17
|
+
* Stand-in for a stripped token. Must not be a letter or a digit, so that the
|
|
18
|
+
* tokenizer treats it as a separator and no new token is produced.
|
|
19
|
+
*/
|
|
20
|
+
const PLACEHOLDER = '…';
|
|
21
|
+
/**
|
|
22
|
+
* CJK characters are letters as far as `\p{L}` is concerned, so a long run of
|
|
23
|
+
* Chinese/Japanese/Korean text with no punctuation matches the same pattern as
|
|
24
|
+
* a base64 blob. Stripping it would silently make that text unsearchable, so
|
|
25
|
+
* such runs are left alone — they are handled by the dedicated CJK index, which
|
|
26
|
+
* splits them per character.
|
|
27
|
+
*/
|
|
28
|
+
const cjkRegExp = /[\u3131-\u314E\u314F-\u3163\uAC00-\uD7A3\u4E00-\u9FCC\u3400-\u4DB5\u3041-\u3096\u30A1-\u30FA]|[\uD840-\uD868][\uDC00-\uDFFF]/u;
|
|
29
|
+
const stripOversizedTokens = (text, pattern, onHit) => {
|
|
30
|
+
const replacements = [];
|
|
31
|
+
let removed = 0;
|
|
32
|
+
const stripped = text.replace(pattern, (match, offset) => {
|
|
33
|
+
if (cjkRegExp.test(match)) {
|
|
34
|
+
return match;
|
|
35
|
+
}
|
|
36
|
+
onHit(match.length);
|
|
37
|
+
replacements.push({
|
|
38
|
+
at: offset - removed,
|
|
39
|
+
delta: match.length - PLACEHOLDER.length,
|
|
40
|
+
});
|
|
41
|
+
removed += match.length - PLACEHOLDER.length;
|
|
42
|
+
return PLACEHOLDER;
|
|
43
|
+
});
|
|
44
|
+
return { stripped, replacements };
|
|
45
|
+
};
|
|
46
|
+
/**
|
|
47
|
+
* `toc[].charIndex` points into `content`, and the search UI uses it to decide
|
|
48
|
+
* which heading a hit belongs to. Shortening `content` without shifting these
|
|
49
|
+
* offsets would attribute hits to the wrong section and link to the wrong
|
|
50
|
+
* anchor, so every offset after a replacement moves back by the same amount.
|
|
51
|
+
*/
|
|
52
|
+
const shiftCharIndex = (charIndex, replacements) => {
|
|
53
|
+
if (charIndex < 0) {
|
|
54
|
+
return charIndex;
|
|
55
|
+
}
|
|
56
|
+
let shift = 0;
|
|
57
|
+
for (const { at, delta } of replacements) {
|
|
58
|
+
if (at >= charIndex) {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
shift += delta;
|
|
62
|
+
}
|
|
63
|
+
return charIndex - shift;
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Keeps pathologically long tokens out of the browser-side search index.
|
|
67
|
+
*
|
|
68
|
+
* Without this, three pages of the connectors documentation — each embedding a
|
|
69
|
+
* base64-encoded SVG icon inside a YAML sample — turned a 1.1 MB index file
|
|
70
|
+
* into ~2 GB of browser memory and crashed the tab with `Out of Memory` as soon
|
|
71
|
+
* as the user opened the search box. Stripping ~20,000 characters (2% of the
|
|
72
|
+
* indexed text) brought that down to 72 MB.
|
|
73
|
+
*/
|
|
74
|
+
export const searchIndexPlugin = ({ maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH, } = {}) => {
|
|
75
|
+
// Mirrors FlexSearch's default word boundary: everything that is neither a
|
|
76
|
+
// letter nor a digit separates tokens.
|
|
77
|
+
const oversizedToken = new RegExp(String.raw `[\p{L}\p{N}]{${maxTokenLength + 1},}`, 'gu');
|
|
78
|
+
return {
|
|
79
|
+
name: 'doom-search-index',
|
|
80
|
+
modifySearchIndexData(pages) {
|
|
81
|
+
let strippedTokens = 0;
|
|
82
|
+
let strippedChars = 0;
|
|
83
|
+
let longest = 0;
|
|
84
|
+
let longestPage = '';
|
|
85
|
+
const affectedPages = [];
|
|
86
|
+
for (const page of pages) {
|
|
87
|
+
let pageTokens = 0;
|
|
88
|
+
const onHit = (length) => {
|
|
89
|
+
pageTokens++;
|
|
90
|
+
strippedTokens++;
|
|
91
|
+
strippedChars += length;
|
|
92
|
+
if (length > longest) {
|
|
93
|
+
longest = length;
|
|
94
|
+
longestPage = page.routePath;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
if (page.content) {
|
|
98
|
+
const { stripped, replacements } = stripOversizedTokens(page.content, oversizedToken, onHit);
|
|
99
|
+
if (replacements.length) {
|
|
100
|
+
page.content = stripped;
|
|
101
|
+
page.toc = page.toc.map((header) => ({
|
|
102
|
+
...header,
|
|
103
|
+
charIndex: shiftCharIndex(header.charIndex, replacements),
|
|
104
|
+
}));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (page.title) {
|
|
108
|
+
page.title = stripOversizedTokens(page.title, oversizedToken, onHit).stripped;
|
|
109
|
+
}
|
|
110
|
+
if (pageTokens) {
|
|
111
|
+
affectedPages.push(page.routePath);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
// Silence is the normal case. When something *was* stripped the author
|
|
115
|
+
// should hear about it: it means machine data is sitting in the prose.
|
|
116
|
+
if (strippedTokens) {
|
|
117
|
+
console.warn(`[doom] search index: stripped ${strippedTokens} token(s) longer than ` +
|
|
118
|
+
`${maxTokenLength} characters (${strippedChars} characters total) from ` +
|
|
119
|
+
`${affectedPages.length} page(s). Longest was ${longest} characters in ` +
|
|
120
|
+
`${longestPage}. Such tokens are usually base64 payloads or credentials ` +
|
|
121
|
+
`pasted into code samples; they make the search index cost quadratic ` +
|
|
122
|
+
`memory in the browser and are not searchable content.`);
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
};
|