@bitbaum/searchkit 0.0.0-stage → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +163 -2
- package/dist/distance.d.ts +24 -0
- package/dist/distance.js +91 -0
- package/dist/engine.d.ts +76 -0
- package/dist/engine.js +350 -0
- package/dist/evaluate.d.ts +46 -0
- package/dist/evaluate.js +127 -0
- package/dist/expand.d.ts +27 -0
- package/dist/expand.js +63 -0
- package/dist/highlight.d.ts +28 -0
- package/dist/highlight.js +95 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/normalize.d.ts +45 -0
- package/dist/normalize.js +103 -0
- package/dist/react/index.d.ts +41 -0
- package/dist/react/index.js +59 -0
- package/dist/typo.d.ts +23 -0
- package/dist/typo.js +22 -0
- package/dist/vocabulary.d.ts +38 -0
- package/dist/vocabulary.js +118 -0
- package/package.json +55 -4
- package/src/distance.ts +93 -0
- package/src/engine.ts +492 -0
- package/src/evaluate.ts +162 -0
- package/src/expand.ts +89 -0
- package/src/highlight.ts +115 -0
- package/src/index.ts +52 -0
- package/src/normalize.ts +127 -0
- package/src/react/index.ts +103 -0
- package/src/typo.ts +31 -0
- package/src/vocabulary.ts +164 -0
package/src/evaluate.ts
ADDED
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prove it: the misspellings people make, generated, and a score for whether
|
|
3
|
+
* search still finds the thing.
|
|
4
|
+
*
|
|
5
|
+
* A typo-tolerance claim is only as good as the test set behind it. This
|
|
6
|
+
* builds that set mechanically from your own titles — every wrong letter that
|
|
7
|
+
* sits next to the right one on a keyboard, every dropped letter, every
|
|
8
|
+
* doubled letter, every swapped pair — so "one slip in a name still finds it"
|
|
9
|
+
* becomes a failing test the day it stops being true, instead of a complaint
|
|
10
|
+
* from someone who could not find an artist.
|
|
11
|
+
*/
|
|
12
|
+
import { words as foldWords } from './normalize.ts';
|
|
13
|
+
import { typoBudget, DEFAULT_TYPO_POLICY, type TypoPolicy } from './typo.ts';
|
|
14
|
+
|
|
15
|
+
/** Physical neighbours on QWERTY and QWERTZ (y and z both neighbour t/u/a/s/x). */
|
|
16
|
+
const NEIGHBOURS: Record<string, string> = {
|
|
17
|
+
q: 'wa',
|
|
18
|
+
w: 'qeas',
|
|
19
|
+
e: 'wrds',
|
|
20
|
+
r: 'etdf',
|
|
21
|
+
t: 'ryfg',
|
|
22
|
+
y: 'tughzas',
|
|
23
|
+
u: 'yihj',
|
|
24
|
+
i: 'uojk',
|
|
25
|
+
o: 'ipkl',
|
|
26
|
+
p: 'ol',
|
|
27
|
+
a: 'qwszy',
|
|
28
|
+
s: 'awedxzy',
|
|
29
|
+
d: 'serfcx',
|
|
30
|
+
f: 'drtgvc',
|
|
31
|
+
g: 'ftyhbv',
|
|
32
|
+
h: 'gyujnb',
|
|
33
|
+
j: 'huikmn',
|
|
34
|
+
k: 'jiolm',
|
|
35
|
+
l: 'kop',
|
|
36
|
+
z: 'asxtuh',
|
|
37
|
+
x: 'zsdc',
|
|
38
|
+
c: 'xdfv',
|
|
39
|
+
v: 'cfgb',
|
|
40
|
+
b: 'vghn',
|
|
41
|
+
n: 'bhjm',
|
|
42
|
+
m: 'njk',
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export type TypoKind = 'neighbour' | 'drop' | 'double' | 'swap';
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The one-slip misspellings of a (folded) word: a neighbouring key instead of
|
|
49
|
+
* a letter, a letter dropped, a letter doubled, two adjacent letters swapped.
|
|
50
|
+
* De-duplicated, and never the word itself.
|
|
51
|
+
*/
|
|
52
|
+
export function typoVariants(
|
|
53
|
+
word: string,
|
|
54
|
+
kinds: readonly TypoKind[] = ['neighbour', 'drop', 'double', 'swap']
|
|
55
|
+
): string[] {
|
|
56
|
+
const s = Array.from(word);
|
|
57
|
+
const out = new Set<string>();
|
|
58
|
+
for (let i = 0; i < s.length; i++) {
|
|
59
|
+
if (kinds.includes('neighbour')) {
|
|
60
|
+
for (const n of NEIGHBOURS[s[i]!] ?? '') {
|
|
61
|
+
out.add([...s.slice(0, i), n, ...s.slice(i + 1)].join(''));
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
if (kinds.includes('drop') && s.length > 1) {
|
|
65
|
+
out.add([...s.slice(0, i), ...s.slice(i + 1)].join(''));
|
|
66
|
+
}
|
|
67
|
+
if (kinds.includes('double')) {
|
|
68
|
+
out.add([...s.slice(0, i + 1), s[i]!, ...s.slice(i + 1)].join(''));
|
|
69
|
+
}
|
|
70
|
+
if (kinds.includes('swap') && i + 1 < s.length && s[i] !== s[i + 1]) {
|
|
71
|
+
out.add([...s.slice(0, i), s[i + 1]!, s[i]!, ...s.slice(i + 2)].join(''));
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
out.delete(word);
|
|
75
|
+
return [...out];
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** A query and what it should find. */
|
|
79
|
+
export interface EvalCase {
|
|
80
|
+
query: string;
|
|
81
|
+
/** Ids that count as the right answer. */
|
|
82
|
+
expect: readonly string[];
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface EvalReport {
|
|
86
|
+
cases: number;
|
|
87
|
+
/** The right answer was first. */
|
|
88
|
+
successAt1: number;
|
|
89
|
+
/** The right answer was in the top 3. */
|
|
90
|
+
successAt3: number;
|
|
91
|
+
/** Mean reciprocal rank of the first right answer (0 when absent). */
|
|
92
|
+
mrr: number;
|
|
93
|
+
/** Share of queries that returned nothing at all. */
|
|
94
|
+
zeroResults: number;
|
|
95
|
+
/** The cases whose answer was not in the top 3, to read. */
|
|
96
|
+
misses: { query: string; expect: readonly string[]; got: readonly string[] }[];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Run every case through `search` (ids, best first) and score it. */
|
|
100
|
+
export async function evaluate(
|
|
101
|
+
cases: readonly EvalCase[],
|
|
102
|
+
search: (query: string) => readonly string[] | Promise<readonly string[]>
|
|
103
|
+
): Promise<EvalReport> {
|
|
104
|
+
let at1 = 0;
|
|
105
|
+
let at3 = 0;
|
|
106
|
+
let rr = 0;
|
|
107
|
+
let zero = 0;
|
|
108
|
+
const misses: EvalReport['misses'] = [];
|
|
109
|
+
for (const c of cases) {
|
|
110
|
+
const got = await search(c.query);
|
|
111
|
+
if (got.length === 0) {
|
|
112
|
+
zero++;
|
|
113
|
+
}
|
|
114
|
+
const rank = got.findIndex(id => c.expect.includes(id));
|
|
115
|
+
if (rank === 0) {
|
|
116
|
+
at1++;
|
|
117
|
+
}
|
|
118
|
+
if (rank >= 0 && rank < 3) {
|
|
119
|
+
at3++;
|
|
120
|
+
} else {
|
|
121
|
+
misses.push({ query: c.query, expect: c.expect, got: got.slice(0, 5) });
|
|
122
|
+
}
|
|
123
|
+
rr += rank >= 0 ? 1 / (rank + 1) : 0;
|
|
124
|
+
}
|
|
125
|
+
const n = cases.length || 1;
|
|
126
|
+
return {
|
|
127
|
+
cases: cases.length,
|
|
128
|
+
successAt1: at1 / n,
|
|
129
|
+
successAt3: at3 / n,
|
|
130
|
+
mrr: rr / n,
|
|
131
|
+
zeroResults: zero / n,
|
|
132
|
+
misses,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Cases from your own names: each name's words, each misspelled once — but
|
|
138
|
+
* only words long enough to be allowed a typo (`typo.ts`), since a 3-letter
|
|
139
|
+
* word with a wrong letter is a different word.
|
|
140
|
+
*/
|
|
141
|
+
export function typoCases(
|
|
142
|
+
names: readonly { id: string; name: string }[],
|
|
143
|
+
options: { policy?: TypoPolicy; perWord?: number; fold?: (s: string) => string[] } = {}
|
|
144
|
+
): EvalCase[] {
|
|
145
|
+
const { policy = DEFAULT_TYPO_POLICY, perWord = Infinity, fold } = options;
|
|
146
|
+
const cases: EvalCase[] = [];
|
|
147
|
+
for (const { id, name } of names) {
|
|
148
|
+
// The search's own words by default: "No-screen" is two words to it, so
|
|
149
|
+
// a slip in "no" must not be generated as "bo-screen".
|
|
150
|
+
const words = fold ? fold(name) : foldWords(name);
|
|
151
|
+
words.forEach((word, i) => {
|
|
152
|
+
if (typoBudget(Array.from(word).length, policy) < 1) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
for (const variant of typoVariants(word).slice(0, perWord)) {
|
|
156
|
+
const query = [...words.slice(0, i), variant, ...words.slice(i + 1)].join(' ');
|
|
157
|
+
cases.push({ query, expect: [id] });
|
|
158
|
+
}
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
return cases;
|
|
162
|
+
}
|
package/src/expand.ts
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typo tolerance for a search that runs somewhere else — a database, an API.
|
|
3
|
+
*
|
|
4
|
+
* Expand each typed word into the collection's words it could mean, and send
|
|
5
|
+
* those along with what was typed. The database then only has to match words
|
|
6
|
+
* exactly or by prefix, which every one can do with an index; the forgiving
|
|
7
|
+
* part happens here, the same way for every backend.
|
|
8
|
+
*
|
|
9
|
+
* And when nothing matched: `didYouMean` names the closest query made of
|
|
10
|
+
* words that exist, to offer as a one-tap fix.
|
|
11
|
+
*/
|
|
12
|
+
import { tokenize, writtenLengths } from './normalize.ts';
|
|
13
|
+
import { DEFAULT_TYPO_POLICY, type TypoPolicy } from './typo.ts';
|
|
14
|
+
import { candidates, type Candidate, type Vocabulary } from './vocabulary.ts';
|
|
15
|
+
|
|
16
|
+
export interface ExpandedWord {
|
|
17
|
+
/** The word as typed, folded. */
|
|
18
|
+
typed: string;
|
|
19
|
+
/** The collection's words it could mean, best first (exact and prefixes included). */
|
|
20
|
+
candidates: Candidate[];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface ExpandOptions {
|
|
24
|
+
policy?: TypoPolicy;
|
|
25
|
+
/** Per typed word, at most this many corrected spellings. Default 5. */
|
|
26
|
+
perWord?: number;
|
|
27
|
+
/** The same site spelling rule the vocabulary was built with (`SearchSpec.normalize`). */
|
|
28
|
+
normalize?: (word: string) => string;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Every typed word with the collection's words it could mean. */
|
|
32
|
+
export function expandQuery(
|
|
33
|
+
query: string,
|
|
34
|
+
vocab: Vocabulary,
|
|
35
|
+
options: ExpandOptions = {}
|
|
36
|
+
): ExpandedWord[] {
|
|
37
|
+
const { policy = DEFAULT_TYPO_POLICY, perWord = 5, normalize = (w: string) => w } = options;
|
|
38
|
+
const typed = writtenLengths(query, normalize);
|
|
39
|
+
return [...typed].map(([word, writtenLength]) => ({
|
|
40
|
+
typed: word,
|
|
41
|
+
candidates: candidates(vocab, word, { prefix: true, policy, limit: 100, writtenLength })
|
|
42
|
+
.filter(c => c.typos > 0 || !c.prefix || c.term === word)
|
|
43
|
+
.slice(0, perWord),
|
|
44
|
+
}));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The words to send to an exact/prefix backend so it forgives typos: each
|
|
49
|
+
* correction that is not simply what was typed. "cqto pots" → ["cato", "post"].
|
|
50
|
+
*/
|
|
51
|
+
export function corrections(query: string, vocab: Vocabulary, options?: ExpandOptions): string[] {
|
|
52
|
+
const out = new Set<string>();
|
|
53
|
+
for (const w of expandQuery(query, vocab, options)) {
|
|
54
|
+
for (const c of w.candidates) {
|
|
55
|
+
if (c.typos > 0) {
|
|
56
|
+
out.add(c.term);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return [...out];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The closest query whose words all exist, or null when every word already
|
|
65
|
+
* does (or none can be corrected). Prefers fewer typos, then the commoner word.
|
|
66
|
+
*/
|
|
67
|
+
export function didYouMean(
|
|
68
|
+
query: string,
|
|
69
|
+
vocab: Vocabulary,
|
|
70
|
+
policy = DEFAULT_TYPO_POLICY
|
|
71
|
+
): string | null {
|
|
72
|
+
const typed = tokenize(query).map(t => t.term);
|
|
73
|
+
if (typed.length === 0) {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
let changed = false;
|
|
77
|
+
const fixed = typed.map(word => {
|
|
78
|
+
if (vocab.counts.has(word)) {
|
|
79
|
+
return word;
|
|
80
|
+
}
|
|
81
|
+
const best = candidates(vocab, word, { prefix: false, policy, limit: 1 })[0];
|
|
82
|
+
if (best) {
|
|
83
|
+
changed = true;
|
|
84
|
+
return best.term;
|
|
85
|
+
}
|
|
86
|
+
return word;
|
|
87
|
+
});
|
|
88
|
+
return changed ? fixed.join(' ') : null;
|
|
89
|
+
}
|
package/src/highlight.ts
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Show WHY a result is there: the parts of its text that matched.
|
|
3
|
+
*
|
|
4
|
+
* From a hit's ranges when you have them (they include typo matches — "cqto"
|
|
5
|
+
* marks "Cato" whole — and mark only the typed start of a prefix match). From
|
|
6
|
+
* the query alone otherwise: every word that STARTS with a typed word, the way
|
|
7
|
+
* the engine matches, never letters in the middle of a word ("lo" marks
|
|
8
|
+
* "local", not "Velo").
|
|
9
|
+
*
|
|
10
|
+
* Plain parts, never HTML: the caller renders them, so a result's own text can
|
|
11
|
+
* never be injected as markup.
|
|
12
|
+
*/
|
|
13
|
+
import { tokenize } from './normalize.ts';
|
|
14
|
+
import type { Range } from './engine.ts';
|
|
15
|
+
|
|
16
|
+
export interface HighlightPart {
|
|
17
|
+
text: string;
|
|
18
|
+
match: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Split `text` into marked and unmarked runs by `ranges` (overlaps merged). */
|
|
22
|
+
export function highlightRanges(text: string, ranges: readonly Range[]): HighlightPart[] {
|
|
23
|
+
if (!text) {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
const sorted = ranges
|
|
27
|
+
.map(r => ({ start: Math.max(0, r.start), end: Math.min(text.length, r.end) }))
|
|
28
|
+
.filter(r => r.end > r.start)
|
|
29
|
+
.sort((a, b) => a.start - b.start);
|
|
30
|
+
const parts: HighlightPart[] = [];
|
|
31
|
+
let at = 0;
|
|
32
|
+
for (const r of sorted) {
|
|
33
|
+
if (r.end <= at) {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
const start = Math.max(r.start, at);
|
|
37
|
+
if (start > at) {
|
|
38
|
+
parts.push({ text: text.slice(at, start), match: false });
|
|
39
|
+
}
|
|
40
|
+
parts.push({ text: text.slice(start, r.end), match: true });
|
|
41
|
+
at = r.end;
|
|
42
|
+
}
|
|
43
|
+
if (at < text.length) {
|
|
44
|
+
parts.push({ text: text.slice(at), match: false });
|
|
45
|
+
}
|
|
46
|
+
return parts;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Mark every word of `text` that starts with a word of `query`, by the typed
|
|
51
|
+
* length. For text that was not part of the search (a snippet fetched later).
|
|
52
|
+
*/
|
|
53
|
+
export function highlightQuery(text: string, query: string): HighlightPart[] {
|
|
54
|
+
const typed = tokenize(query).map(t => t.term);
|
|
55
|
+
if (!text || typed.length === 0) {
|
|
56
|
+
return text ? [{ text, match: false }] : [];
|
|
57
|
+
}
|
|
58
|
+
const ranges: Range[] = [];
|
|
59
|
+
for (const word of tokenize(text)) {
|
|
60
|
+
const longest = typed
|
|
61
|
+
.filter(t => word.term.startsWith(t))
|
|
62
|
+
.sort((a, b) => b.length - a.length)[0];
|
|
63
|
+
if (longest) {
|
|
64
|
+
// Folding can change length (ß → ss); never mark past the word.
|
|
65
|
+
ranges.push({
|
|
66
|
+
start: word.start,
|
|
67
|
+
end: Math.min(word.end, word.start + Array.from(longest).length),
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return highlightRanges(text, ranges);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export interface Snippet {
|
|
75
|
+
/** The excerpt, with "…" where text was cut. */
|
|
76
|
+
text: string;
|
|
77
|
+
/** The ranges that fall inside it, re-based onto `text`. */
|
|
78
|
+
ranges: Range[];
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The part of a long text worth showing for a match: a window that starts a
|
|
83
|
+
* little before the first matched word, cut at word boundaries, with "…"
|
|
84
|
+
* where it was cut. Without a match it is the text's beginning. For a
|
|
85
|
+
* description whose third paragraph is what matched.
|
|
86
|
+
*/
|
|
87
|
+
export function snippet(
|
|
88
|
+
text: string,
|
|
89
|
+
ranges: readonly Range[],
|
|
90
|
+
{ maxLength = 160, lead = 40 }: { maxLength?: number; lead?: number } = {}
|
|
91
|
+
): Snippet {
|
|
92
|
+
if (text.length <= maxLength) {
|
|
93
|
+
return { text, ranges: [...ranges] };
|
|
94
|
+
}
|
|
95
|
+
const first = ranges.reduce((min, r) => Math.min(min, r.start), Infinity);
|
|
96
|
+
let start = Number.isFinite(first) ? Math.max(0, first - lead) : 0;
|
|
97
|
+
if (start > 0) {
|
|
98
|
+
const space = text.indexOf(' ', start);
|
|
99
|
+
start = space >= 0 && space < first ? space + 1 : start;
|
|
100
|
+
}
|
|
101
|
+
let end = Math.min(text.length, start + maxLength);
|
|
102
|
+
if (end < text.length) {
|
|
103
|
+
const space = text.lastIndexOf(' ', end);
|
|
104
|
+
end = space > start ? space : end;
|
|
105
|
+
}
|
|
106
|
+
const before = start > 0 ? '…' : '';
|
|
107
|
+
const after = end < text.length ? '…' : '';
|
|
108
|
+
const shift = before.length - start;
|
|
109
|
+
return {
|
|
110
|
+
text: before + text.slice(start, end) + after,
|
|
111
|
+
ranges: ranges
|
|
112
|
+
.filter(r => r.end > start && r.start < end)
|
|
113
|
+
.map(r => ({ start: Math.max(r.start, start) + shift, end: Math.min(r.end, end) + shift })),
|
|
114
|
+
};
|
|
115
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
export {
|
|
2
|
+
fold,
|
|
3
|
+
tokenize,
|
|
4
|
+
words,
|
|
5
|
+
umlautSpelling,
|
|
6
|
+
joinedPairs,
|
|
7
|
+
writtenLengths,
|
|
8
|
+
type Token,
|
|
9
|
+
} from './normalize.ts';
|
|
10
|
+
export { editDistance, prefixDistance } from './distance.ts';
|
|
11
|
+
export { typoBudget, DEFAULT_TYPO_POLICY, type TypoPolicy } from './typo.ts';
|
|
12
|
+
export {
|
|
13
|
+
createVocabulary,
|
|
14
|
+
candidates,
|
|
15
|
+
type Candidate,
|
|
16
|
+
type CandidateOptions,
|
|
17
|
+
type Vocabulary,
|
|
18
|
+
} from './vocabulary.ts';
|
|
19
|
+
export {
|
|
20
|
+
createSearch,
|
|
21
|
+
search,
|
|
22
|
+
compareHits,
|
|
23
|
+
rerank,
|
|
24
|
+
type Hit,
|
|
25
|
+
type Range,
|
|
26
|
+
type SearchField,
|
|
27
|
+
type SearchOptions,
|
|
28
|
+
type SearchSpec,
|
|
29
|
+
type Searcher,
|
|
30
|
+
} from './engine.ts';
|
|
31
|
+
export {
|
|
32
|
+
highlightRanges,
|
|
33
|
+
highlightQuery,
|
|
34
|
+
snippet,
|
|
35
|
+
type HighlightPart,
|
|
36
|
+
type Snippet,
|
|
37
|
+
} from './highlight.ts';
|
|
38
|
+
export {
|
|
39
|
+
expandQuery,
|
|
40
|
+
corrections,
|
|
41
|
+
didYouMean,
|
|
42
|
+
type ExpandedWord,
|
|
43
|
+
type ExpandOptions,
|
|
44
|
+
} from './expand.ts';
|
|
45
|
+
export {
|
|
46
|
+
typoVariants,
|
|
47
|
+
typoCases,
|
|
48
|
+
evaluate,
|
|
49
|
+
type TypoKind,
|
|
50
|
+
type EvalCase,
|
|
51
|
+
type EvalReport,
|
|
52
|
+
} from './evaluate.ts';
|
package/src/normalize.ts
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text to words, the same way on both sides of a search.
|
|
3
|
+
*
|
|
4
|
+
* Everything a person might type differently from how a title is written must
|
|
5
|
+
* be folded here, once, or the matcher has to forgive it later as a "typo" —
|
|
6
|
+
* which spends the typo budget on something that was not a mistake:
|
|
7
|
+
*
|
|
8
|
+
* - case and accents: "zurich" is "Zürich", "cafe" is "Café";
|
|
9
|
+
* - letters with no decomposition: ß → ss, æ → ae, ø → o, ł → l;
|
|
10
|
+
* - German spelled without umlauts: "Mueller" is "Müller" (handled at
|
|
11
|
+
* indexing: a word with ä/ö/ü is also indexed with ae/oe/ue);
|
|
12
|
+
* - joined and split words: "OrangeCat", "orange_cat", "orange-cat" and
|
|
13
|
+
* "orange cat" are one name (camelCase and separators split; adjacent
|
|
14
|
+
* words are also indexed joined, see `joinedPairs`).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** Letters that NFKD does not decompose, mapped the way their languages spell them in ASCII. */
|
|
18
|
+
const LETTERS: Record<string, string> = {
|
|
19
|
+
ß: 'ss',
|
|
20
|
+
ẞ: 'ss',
|
|
21
|
+
æ: 'ae',
|
|
22
|
+
Æ: 'ae',
|
|
23
|
+
œ: 'oe',
|
|
24
|
+
Œ: 'oe',
|
|
25
|
+
ø: 'o',
|
|
26
|
+
Ø: 'o',
|
|
27
|
+
ł: 'l',
|
|
28
|
+
Ł: 'l',
|
|
29
|
+
đ: 'd',
|
|
30
|
+
Đ: 'd',
|
|
31
|
+
ð: 'd',
|
|
32
|
+
þ: 'th',
|
|
33
|
+
ı: 'i',
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
const SPECIAL = new RegExp(`[${Object.keys(LETTERS).join('')}]`, 'g');
|
|
37
|
+
|
|
38
|
+
/** Casefold and strip accents. "Straße Zürich" → "strasse zurich". */
|
|
39
|
+
export function fold(text: string): string {
|
|
40
|
+
return text
|
|
41
|
+
.replace(SPECIAL, c => LETTERS[c] ?? c)
|
|
42
|
+
.normalize('NFKD')
|
|
43
|
+
.replace(/\p{M}+/gu, '')
|
|
44
|
+
.toLowerCase();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** One word of a text, with where it sits in the original string. */
|
|
48
|
+
export interface Token {
|
|
49
|
+
/** Folded form, the thing compared. */
|
|
50
|
+
term: string;
|
|
51
|
+
/** Position among the text's words, 0-based. */
|
|
52
|
+
index: number;
|
|
53
|
+
/** Offsets into the ORIGINAL text, for highlighting. */
|
|
54
|
+
start: number;
|
|
55
|
+
end: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// A word is a run of letters and digits. camelCase boundaries ("OrangeCat",
|
|
59
|
+
// "myHTMLParser" → my HTML Parser) also split, before case is folded away.
|
|
60
|
+
const WORD = /[\p{L}\p{N}]+/gu;
|
|
61
|
+
const CAMEL =
|
|
62
|
+
/(?<=\p{Ll})(?=\p{Lu})|(?<=\p{Lu})(?=\p{Lu}\p{Ll})|(?<=\p{L})(?=\p{N})|(?<=\p{N})(?=\p{L})/u;
|
|
63
|
+
|
|
64
|
+
/** The words of a text, folded, with their original offsets. */
|
|
65
|
+
export function tokenize(text: string): Token[] {
|
|
66
|
+
const tokens: Token[] = [];
|
|
67
|
+
for (const match of text.matchAll(WORD)) {
|
|
68
|
+
let offset = match.index ?? 0;
|
|
69
|
+
for (const part of match[0].split(CAMEL)) {
|
|
70
|
+
if (part) {
|
|
71
|
+
tokens.push({
|
|
72
|
+
term: fold(part),
|
|
73
|
+
index: tokens.length,
|
|
74
|
+
start: offset,
|
|
75
|
+
end: offset + part.length,
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
offset += part.length;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return tokens;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Just the folded words. */
|
|
85
|
+
export function words(text: string): string[] {
|
|
86
|
+
return tokenize(text).map(t => t.term);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The other spelling of a German word written with umlauts: "müller" →
|
|
91
|
+
* "mueller". Returns null when the word has none. Applied to the ORIGINAL
|
|
92
|
+
* word (before folding, which turns ü into u).
|
|
93
|
+
*/
|
|
94
|
+
export function umlautSpelling(original: string): string | null {
|
|
95
|
+
if (!/[äöüÄÖÜ]/.test(original)) {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
return fold(original.replace(/[äÄ]/g, 'ae').replace(/[öÖ]/g, 'oe').replace(/[üÜ]/g, 'ue'));
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Adjacent words joined: ["orange", "cat", "shop"] → ["orangecat", "catshop"].
|
|
103
|
+
* Indexed alongside the words so a query typed without the space still finds
|
|
104
|
+
* the name typed with it.
|
|
105
|
+
*/
|
|
106
|
+
export function joinedPairs(terms: string[]): string[] {
|
|
107
|
+
const out: string[] = [];
|
|
108
|
+
for (let i = 0; i + 1 < terms.length; i++) {
|
|
109
|
+
out.push(terms[i]! + terms[i + 1]!);
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Each typed word after the site's spelling rule, with how long it was as written. */
|
|
115
|
+
export function writtenLengths(
|
|
116
|
+
query: string,
|
|
117
|
+
normalize: ((word: string) => string) | undefined
|
|
118
|
+
): Map<string, number> {
|
|
119
|
+
const out = new Map<string, number>();
|
|
120
|
+
for (const t of tokenize(query)) {
|
|
121
|
+
const term = normalize ? normalize(t.term) : t.term;
|
|
122
|
+
if (term) {
|
|
123
|
+
out.set(term, Math.max(out.get(term) ?? 0, Array.from(t.term).length));
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
return out;
|
|
127
|
+
}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The React half: search as you type, and the marked text that says why a
|
|
3
|
+
* result is there. Optional — the core has no React in it.
|
|
4
|
+
*/
|
|
5
|
+
import { createElement, Fragment, useEffect, useRef, useState, type ReactNode } from 'react';
|
|
6
|
+
import { highlightQuery, highlightRanges, type HighlightPart } from '../highlight.ts';
|
|
7
|
+
import type { Range } from '../engine.ts';
|
|
8
|
+
|
|
9
|
+
export interface SearchAsYouTypeOptions {
|
|
10
|
+
/**
|
|
11
|
+
* Pause after the last keystroke. 100 ms: under Nielsen's 0.1 s a response
|
|
12
|
+
* still feels instant, and a burst of typing costs one request, not five.
|
|
13
|
+
*/
|
|
14
|
+
debounceMs?: number;
|
|
15
|
+
/** Below this many characters, do not search. Default 1. */
|
|
16
|
+
minLength?: number;
|
|
17
|
+
/** Search at all (e.g. only while a dialog is open). Default true. */
|
|
18
|
+
enabled?: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface SearchAsYouTypeState<R> {
|
|
22
|
+
results: R | null;
|
|
23
|
+
/** A search for the current text is in flight. */
|
|
24
|
+
loading: boolean;
|
|
25
|
+
error: unknown;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Run `search` for the latest text only. Each keystroke aborts the request
|
|
30
|
+
* before it (the signal is passed in), and a response that arrives after a
|
|
31
|
+
* newer one was asked for is dropped — so a slow "ca" can never overwrite a
|
|
32
|
+
* fast "cato".
|
|
33
|
+
*/
|
|
34
|
+
export function useSearchAsYouType<R>(
|
|
35
|
+
text: string,
|
|
36
|
+
search: (text: string, signal: AbortSignal) => Promise<R>,
|
|
37
|
+
options: SearchAsYouTypeOptions = {}
|
|
38
|
+
): SearchAsYouTypeState<R> {
|
|
39
|
+
const { debounceMs = 100, minLength = 1, enabled = true } = options;
|
|
40
|
+
const [state, setState] = useState<SearchAsYouTypeState<R>>({
|
|
41
|
+
results: null,
|
|
42
|
+
loading: false,
|
|
43
|
+
error: null,
|
|
44
|
+
});
|
|
45
|
+
const searchRef = useRef(search);
|
|
46
|
+
searchRef.current = search;
|
|
47
|
+
|
|
48
|
+
useEffect(() => {
|
|
49
|
+
const trimmed = text.trim();
|
|
50
|
+
if (!enabled || trimmed.length < minLength) {
|
|
51
|
+
setState({ results: null, loading: false, error: null });
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const controller = new AbortController();
|
|
55
|
+
setState(s => ({ ...s, loading: true }));
|
|
56
|
+
const timer = setTimeout(() => {
|
|
57
|
+
searchRef
|
|
58
|
+
.current(trimmed, controller.signal)
|
|
59
|
+
.then(results => {
|
|
60
|
+
if (!controller.signal.aborted) {
|
|
61
|
+
setState({ results, loading: false, error: null });
|
|
62
|
+
}
|
|
63
|
+
})
|
|
64
|
+
.catch((error: unknown) => {
|
|
65
|
+
if (!controller.signal.aborted) {
|
|
66
|
+
setState({ results: null, loading: false, error });
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
}, debounceMs);
|
|
70
|
+
return () => {
|
|
71
|
+
clearTimeout(timer);
|
|
72
|
+
controller.abort();
|
|
73
|
+
};
|
|
74
|
+
}, [text, enabled, minLength, debounceMs]);
|
|
75
|
+
|
|
76
|
+
return state;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface HighlightProps {
|
|
80
|
+
text: string;
|
|
81
|
+
/** Mark these ranges (from a hit) … */
|
|
82
|
+
ranges?: readonly Range[];
|
|
83
|
+
/** … or the words that start with what was typed. */
|
|
84
|
+
query?: string;
|
|
85
|
+
/** Class for each <mark>; unstyled by default so it takes your tokens. */
|
|
86
|
+
markClassName?: string;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Text with the matched parts in <mark>. Never renders the text as HTML. */
|
|
90
|
+
export function Highlight({ text, ranges, query = '', markClassName }: HighlightProps): ReactNode {
|
|
91
|
+
const parts: HighlightPart[] = ranges
|
|
92
|
+
? highlightRanges(text, ranges)
|
|
93
|
+
: highlightQuery(text, query);
|
|
94
|
+
return createElement(
|
|
95
|
+
Fragment,
|
|
96
|
+
null,
|
|
97
|
+
...parts.map((part, i) =>
|
|
98
|
+
part.match
|
|
99
|
+
? createElement('mark', { key: i, className: markClassName }, part.text)
|
|
100
|
+
: part.text
|
|
101
|
+
)
|
|
102
|
+
);
|
|
103
|
+
}
|
package/src/typo.ts
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How many typos a word of a given length may carry and still match.
|
|
3
|
+
*
|
|
4
|
+
* The SoundCloud failure — one wrong letter in an artist's name and nothing
|
|
5
|
+
* comes back — is what a budget of zero does. The opposite failure is a short
|
|
6
|
+
* word matching half the dictionary: two edits turn "cat" into "car", "hat",
|
|
7
|
+
* "cut", "at"… So the budget grows with the word, the way Algolia, Typesense
|
|
8
|
+
* and Meilisearch all settled on: none below 4 letters, one from 4, two from 8.
|
|
9
|
+
*
|
|
10
|
+
* Four letters is deliberately the first length with a typo: "cqto" must find
|
|
11
|
+
* "Cato". Pure trigram similarity cannot promise that — "cqto" and "cato"
|
|
12
|
+
* share 2 of 5 trigrams, 0.4, under any useful threshold — which is why this
|
|
13
|
+
* package counts edits instead.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export interface TypoPolicy {
|
|
17
|
+
/** Shortest word allowed one typo. Default 4. */
|
|
18
|
+
oneTypoFrom: number;
|
|
19
|
+
/** Shortest word allowed two typos. Default 8. */
|
|
20
|
+
twoTyposFrom: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export const DEFAULT_TYPO_POLICY: TypoPolicy = { oneTypoFrom: 4, twoTyposFrom: 8 };
|
|
24
|
+
|
|
25
|
+
/** Typos allowed for a query word of this many letters. */
|
|
26
|
+
export function typoBudget(length: number, policy: TypoPolicy = DEFAULT_TYPO_POLICY): number {
|
|
27
|
+
if (length >= policy.twoTyposFrom) {
|
|
28
|
+
return 2;
|
|
29
|
+
}
|
|
30
|
+
return length >= policy.oneTypoFrom ? 1 : 0;
|
|
31
|
+
}
|