@alauda/doom 2.8.0 → 2.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,7 +14,7 @@ import { transformerMetaHighlight, transformerMetaWordHighlight, transformerNota
14
14
  import { difference } from 'es-toolkit';
15
15
  import { glob } from 'tinyglobby';
16
16
  import { cyan } from 'yoctocolors';
17
- import { apiPlugin, autoSidebarPlugin, autoTocPlugin, createTransformerCallouts, directivesPlugin, globalPlugin, mermaidPlugin, permissionPlugin, replacePlugin, } from "../plugins/index.js";
17
+ import { apiPlugin, autoSidebarPlugin, autoTocPlugin, createTransformerCallouts, directivesPlugin, globalPlugin, mermaidPlugin, permissionPlugin, replacePlugin, searchIndexPlugin, } from "../plugins/index.js";
18
18
  import { isExplicitlyUnversioned, normalizeSlash, UNVERSIONED, } from "../shared/index.js";
19
19
  import { pathExists, pkgResolve, resolveStaticConfig } from "../utils/index.js";
20
20
  import { DEFAULT_CONFIG_NAME, DEFAULT_EXTENSIONS, I18N_FILE, SITES_FILE, YAML_EXTENSIONS, } from "./constants.js";
@@ -190,6 +190,7 @@ const getCommonConfig = async ({ config, root, configFilePath, base, version, do
190
190
  localBasePath,
191
191
  force,
192
192
  }),
193
+ searchIndexPlugin(),
193
194
  ].filter(Boolean),
194
195
  search: algoliaOptions ? false : undefined,
195
196
  builderConfig: {
@@ -1,6 +1,7 @@
1
1
  import { visit } from 'unist-util-visit';
2
2
  import { explainUnparsedEmphasis, findUnparsedEmphasis, } from "../remark-lint/no-unparsed-emphasis.js";
3
- import { collectComponents, collectHeadingDepths, } from "../remark-lint/translation-parity/shared.js";
3
+ import { collectComponents, collectHeadingDepths, diffMultiset, } from "../remark-lint/translation-parity/shared.js";
4
+ import { collectProseUrls } from "../remark-lint/translation-parity/url-residue.js";
4
5
  import { countPlaceholders } from "./translate-mask.js";
5
6
  import { parseAttributeElement } from "./translate-segment.js";
6
7
  /**
@@ -140,11 +141,13 @@ const collectTrailingAnchors = (tree) => {
140
141
  /**
141
142
  * The skeleton is the source's, not the model's.
142
143
  *
143
- * Headings and components are structure rather than prose: translating a page
144
- * never changes a heading's level and never adds or removes a component. Both
145
- * are already checked over the whole document by `translation-heading-sequence`
146
- * and `translation-component-multiset` — this runs the *same* collectors, one
147
- * segment at a time.
144
+ * Headings, components and URLs written in prose are structure rather than
145
+ * prose: translating a page never changes a heading's level, never adds or
146
+ * removes a component, and never drops a URL that markdown left as plain text.
147
+ * All three are already checked over the whole document by
148
+ * `translation-heading-sequence`, `translation-component-multiset` and
149
+ * `translation-url-residue` — this runs the *same* collectors, one segment at
150
+ * a time.
148
151
  *
149
152
  * That is not a duplicate of those rules, it is where they become actionable.
150
153
  * Every pairwise rule reports against the document as a whole, with no line
@@ -193,6 +196,26 @@ const structureFindings = (segment, tree, processor) => {
193
196
  : `This segment's translation invented ${got - want} \`<${name}>\` (the source has ${want}, the translation has ${got}).`,
194
197
  });
195
198
  }
199
+ // URLs markdown did not turn into links.
200
+ //
201
+ // The whole-document rule that owns this comparison,
202
+ // `translation-url-residue`, is the one that proved the note above the hard
203
+ // way: it fired on an assembled page, had no segment to be routed to, and
204
+ // failed the document with no repair round — on a false positive, and on
205
+ // every build for a day. Checked here it arrives with a segment attached, so
206
+ // a genuine loss is retranslated instead of being fatal, and the whole-page
207
+ // rule goes back to meaning what the note says it means: assembly did it.
208
+ const expectedUrls = collectProseUrls(source);
209
+ const actualUrls = collectProseUrls(tree);
210
+ const { missing: missingUrls } = diffMultiset(expectedUrls, actualUrls);
211
+ if (missingUrls.length > 0) {
212
+ findings.push({
213
+ rule: 'doom-translate:segment-url-residue',
214
+ reason: `This segment's translation dropped ${missingUrls.length} URL(s) that the source writes out in prose: ${missingUrls
215
+ .map((url) => `\`${url}\``)
216
+ .join(', ')}. They are plain text rather than links, so nothing carries them through for you — reproduce each one exactly as the source spells it.`,
217
+ });
218
+ }
196
219
  // Only meaningful while the headings still correspond one to one; when they
197
220
  // do not, the sequence finding above already says so and pairing by index
198
221
  // would invent a second, misleading one.
@@ -6,4 +6,5 @@ export * from './global/index.ts';
6
6
  export * from './mermaid/index.ts';
7
7
  export * from './permission/index.ts';
8
8
  export * from './replace/index.ts';
9
+ export * from './search-index/index.ts';
9
10
  export * from './shiki/index.ts';
@@ -6,4 +6,5 @@ export * from "./global/index.js";
6
6
  export * from "./mermaid/index.js";
7
7
  export * from "./permission/index.js";
8
8
  export * from "./replace/index.js";
9
+ export * from "./search-index/index.js";
9
10
  export * from "./shiki/index.js";
@@ -0,0 +1,15 @@
1
+ import type { RspressPlugin } from '@rspress/core';
2
+ export interface SearchIndexPluginOptions {
3
+ /** @default 64 */
4
+ maxTokenLength?: number;
5
+ }
6
+ /**
7
+ * Keeps pathologically long tokens out of the browser-side search index.
8
+ *
9
+ * Without this, three pages of the connectors documentation — each embedding a
10
+ * base64-encoded SVG icon inside a YAML sample — turned a 1.1 MB index file
11
+ * into ~2 GB of browser memory and crashed the tab with `Out of Memory` as soon
12
+ * as the user opened the search box. Stripping ~20,000 characters (2% of the
13
+ * indexed text) brought that down to 72 MB.
14
+ */
15
+ export declare const searchIndexPlugin: ({ maxTokenLength, }?: SearchIndexPluginOptions) => RspressPlugin;
@@ -0,0 +1,126 @@
1
+ /**
2
+ * Longest token kept in the search index, in characters.
3
+ *
4
+ * The browser-side search engine (FlexSearch) is configured by rspress with
5
+ * `tokenize: 'full'`, which indexes *every substring* of every token. The cost
6
+ * of a token therefore grows with the square of its length: a 12,000-character
7
+ * token expands into ~70 million substrings and exhausts the tab's memory
8
+ * before the index is finished.
9
+ *
10
+ * Real prose never reaches this length — the longest English words run about
11
+ * 20-30 characters and long code identifiers about 40-50. Tokens above the
12
+ * limit are invariably machine data that nobody searches for: base64 payloads,
13
+ * hashes, JWTs.
14
+ */
15
+ const DEFAULT_MAX_TOKEN_LENGTH = 64;
16
+ /**
17
+ * Stand-in for a stripped token. Must not be a letter or a digit, so that the
18
+ * tokenizer treats it as a separator and no new token is produced.
19
+ */
20
+ const PLACEHOLDER = '…';
21
+ /**
22
+ * CJK characters are letters as far as `\p{L}` is concerned, so a long run of
23
+ * Chinese/Japanese/Korean text with no punctuation matches the same pattern as
24
+ * a base64 blob. Stripping it would silently make that text unsearchable, so
25
+ * such runs are left alone — they are handled by the dedicated CJK index, which
26
+ * splits them per character.
27
+ */
28
+ const cjkRegExp = /[\u3131-\u314E\u314F-\u3163\uAC00-\uD7A3\u4E00-\u9FCC\u3400-\u4DB5\u3041-\u3096\u30A1-\u30FA]|[\uD840-\uD868][\uDC00-\uDFFF]/u;
29
+ const stripOversizedTokens = (text, pattern, onHit) => {
30
+ const replacements = [];
31
+ let removed = 0;
32
+ const stripped = text.replace(pattern, (match, offset) => {
33
+ if (cjkRegExp.test(match)) {
34
+ return match;
35
+ }
36
+ onHit(match.length);
37
+ replacements.push({
38
+ at: offset - removed,
39
+ delta: match.length - PLACEHOLDER.length,
40
+ });
41
+ removed += match.length - PLACEHOLDER.length;
42
+ return PLACEHOLDER;
43
+ });
44
+ return { stripped, replacements };
45
+ };
46
+ /**
47
+ * `toc[].charIndex` points into `content`, and the search UI uses it to decide
48
+ * which heading a hit belongs to. Shortening `content` without shifting these
49
+ * offsets would attribute hits to the wrong section and link to the wrong
50
+ * anchor, so every offset after a replacement moves back by the same amount.
51
+ */
52
+ const shiftCharIndex = (charIndex, replacements) => {
53
+ if (charIndex < 0) {
54
+ return charIndex;
55
+ }
56
+ let shift = 0;
57
+ for (const { at, delta } of replacements) {
58
+ if (at >= charIndex) {
59
+ break;
60
+ }
61
+ shift += delta;
62
+ }
63
+ return charIndex - shift;
64
+ };
65
+ /**
66
+ * Keeps pathologically long tokens out of the browser-side search index.
67
+ *
68
+ * Without this, three pages of the connectors documentation — each embedding a
69
+ * base64-encoded SVG icon inside a YAML sample — turned a 1.1 MB index file
70
+ * into ~2 GB of browser memory and crashed the tab with `Out of Memory` as soon
71
+ * as the user opened the search box. Stripping ~20,000 characters (2% of the
72
+ * indexed text) brought that down to 72 MB.
73
+ */
74
+ export const searchIndexPlugin = ({ maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH, } = {}) => {
75
+ // Mirrors FlexSearch's default word boundary: everything that is neither a
76
+ // letter nor a digit separates tokens.
77
+ const oversizedToken = new RegExp(String.raw `[\p{L}\p{N}]{${maxTokenLength + 1},}`, 'gu');
78
+ return {
79
+ name: 'doom-search-index',
80
+ modifySearchIndexData(pages) {
81
+ let strippedTokens = 0;
82
+ let strippedChars = 0;
83
+ let longest = 0;
84
+ let longestPage = '';
85
+ const affectedPages = [];
86
+ for (const page of pages) {
87
+ let pageTokens = 0;
88
+ const onHit = (length) => {
89
+ pageTokens++;
90
+ strippedTokens++;
91
+ strippedChars += length;
92
+ if (length > longest) {
93
+ longest = length;
94
+ longestPage = page.routePath;
95
+ }
96
+ };
97
+ if (page.content) {
98
+ const { stripped, replacements } = stripOversizedTokens(page.content, oversizedToken, onHit);
99
+ if (replacements.length) {
100
+ page.content = stripped;
101
+ page.toc = page.toc.map((header) => ({
102
+ ...header,
103
+ charIndex: shiftCharIndex(header.charIndex, replacements),
104
+ }));
105
+ }
106
+ }
107
+ if (page.title) {
108
+ page.title = stripOversizedTokens(page.title, oversizedToken, onHit).stripped;
109
+ }
110
+ if (pageTokens) {
111
+ affectedPages.push(page.routePath);
112
+ }
113
+ }
114
+ // Silence is the normal case. When something *was* stripped the author
115
+ // should hear about it: it means machine data is sitting in the prose.
116
+ if (strippedTokens) {
117
+ console.warn(`[doom] search index: stripped ${strippedTokens} token(s) longer than ` +
118
+ `${maxTokenLength} characters (${strippedChars} characters total) from ` +
119
+ `${affectedPages.length} page(s). Longest was ${longest} characters in ` +
120
+ `${longestPage}. Such tokens are usually base64 payloads or credentials ` +
121
+ `pasted into code samples; they make the search index cost quadratic ` +
122
+ `memory in the browser and are not searchable content.`);
123
+ }
124
+ },
125
+ };
126
+ };
@@ -19,11 +19,108 @@ import { currentPair, diffMultiset } from "./shared.js";
19
19
  * between prose and inline code. Those are identifiers, not links, and a guard
20
20
  * that cries wolf about them is worse than no guard at all.
21
21
  *
22
- * Honest caveat: measured exposure on this corpus is zero — all seven bare URLs
23
- * carry `https://`, which makes them link nodes, protected structurally. This
24
- * is a guard for a case nothing else covers, not a demonstrated detector.
22
+ * Honest caveat, kept because it turned out to matter: when this shipped,
23
+ * measured exposure on the corpus was zero — all seven bare URLs carried
24
+ * `https://`, which makes them link nodes, protected structurally. It was a
25
+ * guard for a case nothing else covered, not a demonstrated detector. Its
26
+ * first meeting with real prose was a false positive that took a build down
27
+ * for a day; what that cost, and what it changed, is written up on
28
+ * {@link BARE_URL} and {@link HAS_HOST}.
25
29
  */
26
- const BARE_URL = /\b(?:[a-z][a-z0-9+.-]*:\/\/|www\.)\S+/gi;
30
+ /**
31
+ * A URL runs to the first character a URL cannot contain.
32
+ *
33
+ * This used to be `\S+`, "everything up to the next space" — and Chinese does
34
+ * not write spaces. On the translation side the match therefore swallowed the
35
+ * rest of the sentence: `(https://)和路径` yielded `https://)和路径`, which
36
+ * could not equal anything the English side produced no matter how faithful
37
+ * the translation was. `[!-~]` is printable ASCII, which is what a URL is made
38
+ * of, so a match now ends at the first full-width bracket or Han character.
39
+ *
40
+ * `[^\s\P{ASCII}]` reads as a double negative because that is the only way to
41
+ * say it without an opaque code-point range: not whitespace, and not outside
42
+ * ASCII.
43
+ *
44
+ * The letters are spelled out rather than left to an `i` flag, and that is not
45
+ * style. Under `iu` the engine case-folds before testing the class, and two
46
+ * ASCII letters fold to non-ASCII ones — `s` to `ſ` (U+017F), `k` to `K`
47
+ * (U+212A) — so `\P{ASCII}` would swallow them and the negated class would
48
+ * *reject* `s` and `k`. Caught by the one test with an `s` mid-URL:
49
+ * `ftp://h/Foo_(disambiguation)` matched as far as `ftp://h/Foo_(di`.
50
+ *
51
+ * That misfire is not hypothetical and it was not cheap. `connectors-operator`
52
+ * release-1.14 was red on every run for 22 hours over one line of prose —
53
+ * `includes the correct protocol (https://) and path` — where nothing had been
54
+ * lost at all: the translation said the same thing with full-width brackets.
55
+ * And because this rule reports against the whole page, the assembled document
56
+ * could not attribute the finding to any segment and failed outright without a
57
+ * repair round. A misfire here does not cost "one finding to read".
58
+ *
59
+ * The price of the narrowing, stated plainly: an internationalised host
60
+ * written out in prose (`https://例え.jp`) is no longer detected. That is the
61
+ * right trade — the sources are English technical documentation, and a
62
+ * detector that cannot agree with its own translation is not a detector.
63
+ *
64
+ * The sibling rule made the opposite choice, and deliberately:
65
+ * `translation-link-isomorphism` *reports* the full-width stop an autolink
66
+ * absorbed rather than normalising it away, because there the punctuation ends
67
+ * up inside a real `href` and the link genuinely breaks. Prose has no `href`
68
+ * to break — `(ftp://h/x)` reads exactly as intended — so here the two
69
+ * spellings are made comparable instead of being reported.
70
+ */
71
+ const BARE_URL = /\b(?:[a-zA-Z][a-zA-Z0-9+.-]*:\/\/|[wW]{3}\.)[^\s\P{ASCII}]+/gu;
72
+ /**
73
+ * Punctuation that ends the sentence rather than the URL.
74
+ *
75
+ * `(see ftp://h/x)` and `fetch ftp://h/x.tar.` both put a character in the
76
+ * match that the writer did not mean as part of the address, and the
77
+ * translation — which repunctuates, because that is what translating prose
78
+ * does — will not have it. Trimmed the way GFM's autolink extension trims it,
79
+ * which is also the way a reader would.
80
+ *
81
+ * Only ASCII forms are listed: {@link BARE_URL} cannot match anything else, so
82
+ * a full-width bracket here would be a branch that can never be taken.
83
+ */
84
+ const TRAILING_PUNCTUATION = /[!"'*,.:;?_~]+$/u;
85
+ /**
86
+ * Closing brackets, and the opener each needs in order to belong to the URL.
87
+ *
88
+ * So `ftp://h/Foo_(disambiguation)` keeps its parenthesis — the URL opened it
89
+ * — and `(see ftp://h/x)` does not.
90
+ */
91
+ const BRACKETS = new Map([
92
+ [')', '('],
93
+ [']', '['],
94
+ ['}', '{'],
95
+ ]);
96
+ const occurrences = (value, character) => value.split(character).length - 1;
97
+ const trimSentence = (url) => {
98
+ let value = url;
99
+ for (;;) {
100
+ const before = value;
101
+ value = value.replace(TRAILING_PUNCTUATION, '');
102
+ const closing = value.at(-1);
103
+ const opening = closing == null ? undefined : BRACKETS.get(closing);
104
+ if (closing != null &&
105
+ opening != null &&
106
+ occurrences(value, closing) > occurrences(value, opening)) {
107
+ value = value.slice(0, -1);
108
+ }
109
+ if (value === before) {
110
+ return value;
111
+ }
112
+ }
113
+ };
114
+ /**
115
+ * A scheme with nothing after it is not a URL.
116
+ *
117
+ * `https://` on its own is prose — a sentence naming the protocol, as in "the
118
+ * address must include the protocol (https://)". Requiring a host is what
119
+ * tells the two apart, and it gives up nothing this rule was protecting: an
120
+ * address with no host cannot be fetched, so it cannot be a link a translation
121
+ * has to carry through.
122
+ */
123
+ const HAS_HOST = /(?::\/\/|^[wW]{3}\.)[^\s\P{ASCII}]/u;
27
124
  /**
28
125
  * Text inside a link is not prose for this purpose.
29
126
  *
@@ -49,7 +146,10 @@ export const collectProseUrls = (tree) => {
49
146
  return;
50
147
  }
51
148
  for (const [match] of node.value.matchAll(BARE_URL)) {
52
- found.push(match);
149
+ const url = trimSentence(match);
150
+ if (HAS_HOST.test(url)) {
151
+ found.push(url);
152
+ }
53
153
  }
54
154
  });
55
155
  return found;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@alauda/doom",
3
- "version": "2.8.0",
3
+ "version": "2.8.2",
4
4
  "type": "module",
5
5
  "description": "Doctor Doom making docs.",
6
6
  "repository": {