@bitbaum/searchkit 0.0.0-stage → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +163 -2
- package/dist/distance.d.ts +24 -0
- package/dist/distance.js +91 -0
- package/dist/engine.d.ts +76 -0
- package/dist/engine.js +350 -0
- package/dist/evaluate.d.ts +46 -0
- package/dist/evaluate.js +127 -0
- package/dist/expand.d.ts +27 -0
- package/dist/expand.js +63 -0
- package/dist/highlight.d.ts +28 -0
- package/dist/highlight.js +95 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/normalize.d.ts +45 -0
- package/dist/normalize.js +103 -0
- package/dist/react/index.d.ts +41 -0
- package/dist/react/index.js +59 -0
- package/dist/typo.d.ts +23 -0
- package/dist/typo.js +22 -0
- package/dist/vocabulary.d.ts +38 -0
- package/dist/vocabulary.js +118 -0
- package/package.json +55 -4
- package/src/distance.ts +93 -0
- package/src/engine.ts +492 -0
- package/src/evaluate.ts +162 -0
- package/src/expand.ts +89 -0
- package/src/highlight.ts +115 -0
- package/src/index.ts +52 -0
- package/src/normalize.ts +127 -0
- package/src/react/index.ts +103 -0
- package/src/typo.ts +31 -0
- package/src/vocabulary.ts +164 -0
package/dist/engine.js
ADDED
|
@@ -0,0 +1,350 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search a list: find what was meant, and put it first.
|
|
3
|
+
*
|
|
4
|
+
* Matching is per typed word. Each word may match a word of the record
|
|
5
|
+
* exactly, as its start (the reader is still typing), within the typo budget
|
|
6
|
+
* (`typo.ts`), or through a second spelling indexed with the record (German
|
|
7
|
+
* without umlauts, two words written as one). Every typed word must match;
|
|
8
|
+
* a word that matches nothing anywhere is set aside, and when no record has
|
|
9
|
+
* all the rest, the last words are dropped one at a time — so a query with a
|
|
10
|
+
* word too many, or one too mangled to correct, still lands somewhere.
|
|
11
|
+
*
|
|
12
|
+
* Ranking is a tie-break, not a weighted sum. Sums need tuning and drift:
|
|
13
|
+
* a long description outscores a short exact title. Each rule below only
|
|
14
|
+
* decides between records the rules above it called equal:
|
|
15
|
+
*
|
|
16
|
+
* 1. typos — fewer typos; a wrong FIRST letter counts double here
|
|
17
|
+
* (it still matches — Meilisearch refuses it outright, which
|
|
18
|
+
* loses every name whose first letter was mistyped)
|
|
19
|
+
* 2. words — more of the typed words matched (differs only once the
|
|
20
|
+
* query has been relaxed)
|
|
21
|
+
* 3. attribute — matched in a more important field (the order you list them)
|
|
22
|
+
* 4. exact — the whole field typed exactly; then the whole field
|
|
23
|
+
* matched (typos and prefixes allowed: "lki" is all of
|
|
24
|
+
* "Loki"); then whole words over prefixes
|
|
25
|
+
* 5. proximity — the matched words sit closer together, in order
|
|
26
|
+
* 6. position — the match is nearer the start of the field
|
|
27
|
+
* 7. custom — your own `rank` (popularity, recency), higher first
|
|
28
|
+
* 8. length — the shorter field, which is the more specific match
|
|
29
|
+
*
|
|
30
|
+
* It is Algolia's published default order, adjusted for short titles and
|
|
31
|
+
* names, which is what people search a site for.
|
|
32
|
+
*/
|
|
33
|
+
import { joinedPairs, tokenize, umlautSpelling, writtenLengths } from "./normalize.js";
|
|
34
|
+
import { DEFAULT_TYPO_POLICY } from "./typo.js";
|
|
35
|
+
import { candidates, createVocabulary } from "./vocabulary.js";
|
|
36
|
+
function fieldTexts(field, item) {
|
|
37
|
+
const v = field.get(item);
|
|
38
|
+
if (v === null || v === undefined) {
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
41
|
+
return (typeof v === 'string' ? [v] : [...v]).filter((s) => typeof s === 'string' && s.length > 0);
|
|
42
|
+
}
|
|
43
|
+
function buildIndex(items, spec) {
|
|
44
|
+
const n = spec.normalize ?? ((w) => w);
|
|
45
|
+
const postings = new Map();
|
|
46
|
+
const words = [];
|
|
47
|
+
const fieldTerms = [];
|
|
48
|
+
const post = (term, p) => {
|
|
49
|
+
const list = postings.get(term);
|
|
50
|
+
if (list) {
|
|
51
|
+
list.push(p);
|
|
52
|
+
}
|
|
53
|
+
else {
|
|
54
|
+
postings.set(term, [p]);
|
|
55
|
+
}
|
|
56
|
+
words.push(term);
|
|
57
|
+
};
|
|
58
|
+
items.forEach((item, doc) => {
|
|
59
|
+
const perField = [];
|
|
60
|
+
spec.fields.forEach((field, f) => {
|
|
61
|
+
const terms = [];
|
|
62
|
+
// Several values of one field (tags, aliases) are separate texts; a
|
|
63
|
+
// gap of 1000 positions keeps proximity from bridging them.
|
|
64
|
+
fieldTexts(field, item).forEach((text, part) => {
|
|
65
|
+
const tokens = tokenize(text);
|
|
66
|
+
const base = part * 1000;
|
|
67
|
+
const whole = tokens.length === 1;
|
|
68
|
+
for (const t of tokens) {
|
|
69
|
+
const term = n(t.term);
|
|
70
|
+
post(term, {
|
|
71
|
+
doc,
|
|
72
|
+
field: f,
|
|
73
|
+
position: base + t.index,
|
|
74
|
+
start: t.start,
|
|
75
|
+
end: t.end,
|
|
76
|
+
alternate: false,
|
|
77
|
+
whole,
|
|
78
|
+
});
|
|
79
|
+
terms.push(term);
|
|
80
|
+
const spelt = umlautSpelling(text.slice(t.start, t.end));
|
|
81
|
+
const other = spelt ? n(spelt) : null;
|
|
82
|
+
if (other && other !== term) {
|
|
83
|
+
post(other, {
|
|
84
|
+
doc,
|
|
85
|
+
field: f,
|
|
86
|
+
position: base + t.index,
|
|
87
|
+
start: t.start,
|
|
88
|
+
end: t.end,
|
|
89
|
+
alternate: true,
|
|
90
|
+
whole,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
joinedPairs(tokens.map(t => n(t.term))).forEach((term, i) => {
|
|
95
|
+
const a = tokens[i];
|
|
96
|
+
const b = tokens[i + 1];
|
|
97
|
+
post(term, {
|
|
98
|
+
doc,
|
|
99
|
+
field: f,
|
|
100
|
+
position: base + a.index,
|
|
101
|
+
start: a.start,
|
|
102
|
+
end: b.end,
|
|
103
|
+
alternate: true,
|
|
104
|
+
whole: tokens.length === 2,
|
|
105
|
+
bridged: a.end < b.start,
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
perField.push(terms);
|
|
110
|
+
});
|
|
111
|
+
fieldTerms.push(perField);
|
|
112
|
+
});
|
|
113
|
+
return { vocab: createVocabulary(words), postings, fieldTerms };
|
|
114
|
+
}
|
|
115
|
+
const better = (a, b) => a.rankTypos - b.rankTypos ||
|
|
116
|
+
a.field - b.field ||
|
|
117
|
+
Number(a.prefix) - Number(b.prefix) ||
|
|
118
|
+
a.position - b.position;
|
|
119
|
+
/**
|
|
120
|
+
* Index `items` once; search them as often as you like. Rebuild when the list
|
|
121
|
+
* changes — indexing is linear and cheap next to a network round trip.
|
|
122
|
+
*/
|
|
123
|
+
export function createSearch(items, spec) {
|
|
124
|
+
const index = buildIndex(items, spec);
|
|
125
|
+
const policy = spec.typos ?? DEFAULT_TYPO_POLICY;
|
|
126
|
+
const prefixMode = spec.prefix ?? 'all';
|
|
127
|
+
// Letters each typed word had before `normalize`, for the typo budget; set
|
|
128
|
+
// per search (searching is synchronous).
|
|
129
|
+
let written = new Map();
|
|
130
|
+
function landingsFor(typed, allowPrefix) {
|
|
131
|
+
const best = new Map();
|
|
132
|
+
const options = { prefix: allowPrefix, policy, limit: 200, writtenLength: written.get(typed) };
|
|
133
|
+
for (const c of candidates(index.vocab, typed, options)) {
|
|
134
|
+
for (const p of index.postings.get(c.term) ?? []) {
|
|
135
|
+
const typedLength = Array.from(typed).length;
|
|
136
|
+
const firstSlip = c.typos > 0 && Array.from(c.term)[0] !== Array.from(typed)[0];
|
|
137
|
+
const landing = {
|
|
138
|
+
typos: c.typos,
|
|
139
|
+
rankTypos: c.typos + (firstSlip ? 1 : 0) + (p.bridged ? 1 : 0),
|
|
140
|
+
field: p.field,
|
|
141
|
+
position: p.position,
|
|
142
|
+
prefix: c.prefix,
|
|
143
|
+
start: p.start,
|
|
144
|
+
end: p.end,
|
|
145
|
+
markEnd: c.prefix && !p.alternate ? Math.min(p.end, p.start + typedLength) : p.end,
|
|
146
|
+
};
|
|
147
|
+
const seen = best.get(p.doc);
|
|
148
|
+
if (!seen || better(landing, seen) < 0) {
|
|
149
|
+
best.set(p.doc, landing);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return best;
|
|
154
|
+
}
|
|
155
|
+
function run(allTerms, typingLast, relax) {
|
|
156
|
+
const allLandings = allTerms.map((term, i) => landingsFor(term, prefixMode === 'all' || (prefixMode === 'last' && typingLast && i === allTerms.length - 1)));
|
|
157
|
+
// A word that matches nothing anywhere ("pen" for "open", too short for a
|
|
158
|
+
// typo) cannot help, so with `relax` it stops being required rather than
|
|
159
|
+
// emptying the whole result.
|
|
160
|
+
const keep = allTerms.map((_, i) => !relax || allLandings[i].size > 0);
|
|
161
|
+
const terms = allTerms.filter((_, i) => keep[i]);
|
|
162
|
+
const perTerm = allLandings.filter((_, i) => keep[i]);
|
|
163
|
+
const hits = [];
|
|
164
|
+
const first = perTerm[0];
|
|
165
|
+
if (!first) {
|
|
166
|
+
return hits;
|
|
167
|
+
}
|
|
168
|
+
for (const doc of first.keys()) {
|
|
169
|
+
const landings = perTerm.map(m => m.get(doc));
|
|
170
|
+
if (landings.some(l => !l)) {
|
|
171
|
+
continue;
|
|
172
|
+
}
|
|
173
|
+
const ls = landings;
|
|
174
|
+
const typos = ls.reduce((s, l) => s + l.typos, 0);
|
|
175
|
+
const rankTypos = ls.reduce((s, l) => s + l.rankTypos, 0);
|
|
176
|
+
const field = Math.min(...ls.map(l => l.field));
|
|
177
|
+
const typedWhole = index.fieldTerms[doc].some(fieldWords => fieldWords.length === terms.length && fieldWords.every((w, i) => w === terms[i]));
|
|
178
|
+
const exactWords = ls.filter(l => !l.prefix && l.typos === 0).length;
|
|
179
|
+
// Every word of the matched field was matched: "lki" is the WHOLE of
|
|
180
|
+
// "Loki", but only the first word of "Loki — Capability Layer". And
|
|
181
|
+
// every typed word landed there: for "gallium ch", "ch" alone covering
|
|
182
|
+
// a title "Chalco" (gallium only in its description) is not the name.
|
|
183
|
+
const fieldLength = index.fieldTerms[doc][field].length;
|
|
184
|
+
const coversField = ls.every(l => l.field === field) && new Set(ls.map(l => l.position)).size >= fieldLength;
|
|
185
|
+
let proximity = 0;
|
|
186
|
+
for (let i = 1; i < ls.length; i++) {
|
|
187
|
+
const a = ls[i - 1];
|
|
188
|
+
const b = ls[i];
|
|
189
|
+
const gap = a.field === b.field ? b.position - a.position : 8;
|
|
190
|
+
// In order and adjacent costs 0; out of order or apart costs more, capped.
|
|
191
|
+
proximity += gap === 1 ? 0 : Math.min(gap > 0 ? gap : Math.abs(gap) + 1, 8);
|
|
192
|
+
}
|
|
193
|
+
const position = Math.min(...ls.filter(l => l.field === field).map(l => l.position % 1000));
|
|
194
|
+
const item = items[doc];
|
|
195
|
+
const matches = {};
|
|
196
|
+
for (const l of ls) {
|
|
197
|
+
const name = spec.fields[l.field].name;
|
|
198
|
+
(matches[name] ??= []).push({ start: l.start, end: l.markEnd });
|
|
199
|
+
}
|
|
200
|
+
hits.push({
|
|
201
|
+
item,
|
|
202
|
+
index: doc,
|
|
203
|
+
words: terms.length,
|
|
204
|
+
typos,
|
|
205
|
+
matches,
|
|
206
|
+
score: [
|
|
207
|
+
rankTypos,
|
|
208
|
+
-terms.length,
|
|
209
|
+
field,
|
|
210
|
+
typedWhole ? -1000 : coversField ? -500 - exactWords : -exactWords,
|
|
211
|
+
proximity,
|
|
212
|
+
position,
|
|
213
|
+
-(spec.rank?.(item) ?? 0),
|
|
214
|
+
fieldLength,
|
|
215
|
+
doc,
|
|
216
|
+
],
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
return hits;
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
vocabulary: index.vocab,
|
|
223
|
+
search(query, options = {}) {
|
|
224
|
+
const { limit = Infinity, relax = true } = options;
|
|
225
|
+
written = writtenLengths(query, spec.normalize);
|
|
226
|
+
const terms = [...written.keys()];
|
|
227
|
+
if (terms.length === 0) {
|
|
228
|
+
return [];
|
|
229
|
+
}
|
|
230
|
+
const typingLast = !/\s$/.test(query);
|
|
231
|
+
let hits = run(terms, typingLast, relax);
|
|
232
|
+
// Written apart, meant together ("orange cat" for "Orangecat"), or the
|
|
233
|
+
// other way round ("orangecat" for "Orange Cat" is indexed as a joined
|
|
234
|
+
// pair already; a joined word the index has never seen is split where
|
|
235
|
+
// both halves are words it has).
|
|
236
|
+
if (hits.length === 0 && terms.length > 1) {
|
|
237
|
+
hits = run(joinAdjacent(terms, index.vocab), typingLast, relax);
|
|
238
|
+
}
|
|
239
|
+
if (hits.length === 0) {
|
|
240
|
+
const split = splitUnknown(terms, index.vocab);
|
|
241
|
+
if (split) {
|
|
242
|
+
hits = run(split, typingLast, relax);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
// No item has every word: keep as many as some item has, whichever
|
|
246
|
+
// they are. "jon smith" finds both Jon Snow and John Smith; dropping
|
|
247
|
+
// only the LAST word would lose John Smith.
|
|
248
|
+
if (relax) {
|
|
249
|
+
for (let keep = terms.length - 1; hits.length === 0 && keep >= 1; keep--) {
|
|
250
|
+
const best = new Map();
|
|
251
|
+
for (const subset of combinations(terms.length, keep)) {
|
|
252
|
+
const typing = typingLast && subset[subset.length - 1] === terms.length - 1;
|
|
253
|
+
for (const hit of run(subset.map(i => terms[i]), typing, relax)) {
|
|
254
|
+
const seen = best.get(hit.index);
|
|
255
|
+
if (!seen || compareHits(hit, seen) < 0) {
|
|
256
|
+
best.set(hit.index, hit);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
hits = [...best.values()];
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return hits.sort(compareHits).slice(0, limit);
|
|
264
|
+
},
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Index sets of size k from 0..n-1, in order; past 8 words only the leading
|
|
269
|
+
* ones (a pasted sentence is not worth 2^n searches).
|
|
270
|
+
*/
|
|
271
|
+
function combinations(n, k) {
|
|
272
|
+
if (n > 8) {
|
|
273
|
+
return [Array.from({ length: k }, (_, i) => i)];
|
|
274
|
+
}
|
|
275
|
+
const out = [];
|
|
276
|
+
const pick = (start, chosen) => {
|
|
277
|
+
if (chosen.length === k) {
|
|
278
|
+
out.push(chosen);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
for (let i = start; i <= n - (k - chosen.length); i++) {
|
|
282
|
+
pick(i + 1, [...chosen, i]);
|
|
283
|
+
}
|
|
284
|
+
};
|
|
285
|
+
pick(0, []);
|
|
286
|
+
return out;
|
|
287
|
+
}
|
|
288
|
+
/** Join each adjacent pair the vocabulary knows as one word. */
|
|
289
|
+
function joinAdjacent(terms, vocab) {
|
|
290
|
+
const out = [];
|
|
291
|
+
for (let i = 0; i < terms.length; i++) {
|
|
292
|
+
const joined = i + 1 < terms.length ? terms[i] + terms[i + 1] : null;
|
|
293
|
+
if (joined && vocab.counts.has(joined)) {
|
|
294
|
+
out.push(joined);
|
|
295
|
+
i++;
|
|
296
|
+
}
|
|
297
|
+
else {
|
|
298
|
+
out.push(terms[i]);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return out;
|
|
302
|
+
}
|
|
303
|
+
/** Split each unknown word at the first point where both halves are known words. */
|
|
304
|
+
function splitUnknown(terms, vocab) {
|
|
305
|
+
let changed = false;
|
|
306
|
+
const out = terms.flatMap(term => {
|
|
307
|
+
if (vocab.counts.has(term)) {
|
|
308
|
+
return [term];
|
|
309
|
+
}
|
|
310
|
+
const letters = Array.from(term);
|
|
311
|
+
for (let i = 2; i <= letters.length - 2; i++) {
|
|
312
|
+
const a = letters.slice(0, i).join('');
|
|
313
|
+
const b = letters.slice(i).join('');
|
|
314
|
+
if (vocab.counts.has(a) && vocab.counts.has(b)) {
|
|
315
|
+
changed = true;
|
|
316
|
+
return [a, b];
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return [term];
|
|
320
|
+
});
|
|
321
|
+
return changed ? out : null;
|
|
322
|
+
}
|
|
323
|
+
/** Sort order for hits: the tie-break values, compared left to right. */
|
|
324
|
+
export function compareHits(a, b) {
|
|
325
|
+
for (let i = 0; i < a.score.length; i++) {
|
|
326
|
+
const d = (a.score[i] ?? 0) - (b.score[i] ?? 0);
|
|
327
|
+
if (d !== 0) {
|
|
328
|
+
return d;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
return 0;
|
|
332
|
+
}
|
|
333
|
+
/** Search a list once, without keeping an index. */
|
|
334
|
+
export function search(items, spec, query, options) {
|
|
335
|
+
return createSearch(items, spec).search(query, options);
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* Rank rows another engine already found — a database, an API — by what was
|
|
339
|
+
* typed. Rows the ranker cannot place (they matched on a field you did not
|
|
340
|
+
* pass, or through stemming) keep their original order after the ranked
|
|
341
|
+
* ones: still results, just later ones.
|
|
342
|
+
*/
|
|
343
|
+
export function rerank(rows, spec, query) {
|
|
344
|
+
const hits = createSearch(rows, spec).search(query);
|
|
345
|
+
const placed = new Set(hits.map(h => h.index));
|
|
346
|
+
return [
|
|
347
|
+
...hits.map(hit => ({ item: hit.item, hit })),
|
|
348
|
+
...rows.filter((_, i) => !placed.has(i)).map(item => ({ item, hit: null })),
|
|
349
|
+
];
|
|
350
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { type TypoPolicy } from './typo.ts';
|
|
2
|
+
export type TypoKind = 'neighbour' | 'drop' | 'double' | 'swap';
|
|
3
|
+
/**
|
|
4
|
+
* The one-slip misspellings of a (folded) word: a neighbouring key instead of
|
|
5
|
+
* a letter, a letter dropped, a letter doubled, two adjacent letters swapped.
|
|
6
|
+
* De-duplicated, and never the word itself.
|
|
7
|
+
*/
|
|
8
|
+
export declare function typoVariants(word: string, kinds?: readonly TypoKind[]): string[];
|
|
9
|
+
/** A query and what it should find. */
|
|
10
|
+
export interface EvalCase {
|
|
11
|
+
query: string;
|
|
12
|
+
/** Ids that count as the right answer. */
|
|
13
|
+
expect: readonly string[];
|
|
14
|
+
}
|
|
15
|
+
export interface EvalReport {
|
|
16
|
+
cases: number;
|
|
17
|
+
/** The right answer was first. */
|
|
18
|
+
successAt1: number;
|
|
19
|
+
/** The right answer was in the top 3. */
|
|
20
|
+
successAt3: number;
|
|
21
|
+
/** Mean reciprocal rank of the first right answer (0 when absent). */
|
|
22
|
+
mrr: number;
|
|
23
|
+
/** Share of queries that returned nothing at all. */
|
|
24
|
+
zeroResults: number;
|
|
25
|
+
/** The cases whose answer was not in the top 3, to read. */
|
|
26
|
+
misses: {
|
|
27
|
+
query: string;
|
|
28
|
+
expect: readonly string[];
|
|
29
|
+
got: readonly string[];
|
|
30
|
+
}[];
|
|
31
|
+
}
|
|
32
|
+
/** Run every case through `search` (ids, best first) and score it. */
|
|
33
|
+
export declare function evaluate(cases: readonly EvalCase[], search: (query: string) => readonly string[] | Promise<readonly string[]>): Promise<EvalReport>;
|
|
34
|
+
/**
|
|
35
|
+
* Cases from your own names: each name's words, each misspelled once — but
|
|
36
|
+
* only words long enough to be allowed a typo (`typo.ts`), since a 3-letter
|
|
37
|
+
* word with a wrong letter is a different word.
|
|
38
|
+
*/
|
|
39
|
+
export declare function typoCases(names: readonly {
|
|
40
|
+
id: string;
|
|
41
|
+
name: string;
|
|
42
|
+
}[], options?: {
|
|
43
|
+
policy?: TypoPolicy;
|
|
44
|
+
perWord?: number;
|
|
45
|
+
fold?: (s: string) => string[];
|
|
46
|
+
}): EvalCase[];
|
package/dist/evaluate.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prove it: the misspellings people make, generated, and a score for whether
|
|
3
|
+
* search still finds the thing.
|
|
4
|
+
*
|
|
5
|
+
* A typo-tolerance claim is only as good as the test set behind it. This
|
|
6
|
+
* builds that set mechanically from your own titles — every wrong letter that
|
|
7
|
+
* sits next to the right one on a keyboard, every dropped letter, every
|
|
8
|
+
* doubled letter, every swapped pair — so "one slip in a name still finds it"
|
|
9
|
+
* becomes a failing test the day it stops being true, instead of a complaint
|
|
10
|
+
* from someone who could not find an artist.
|
|
11
|
+
*/
|
|
12
|
+
import { words as foldWords } from "./normalize.js";
|
|
13
|
+
import { typoBudget, DEFAULT_TYPO_POLICY } from "./typo.js";
|
|
14
|
+
/** Physical neighbours on QWERTY and QWERTZ (y and z both neighbour t/u/a/s/x). */
|
|
15
|
+
const NEIGHBOURS = {
|
|
16
|
+
q: 'wa',
|
|
17
|
+
w: 'qeas',
|
|
18
|
+
e: 'wrds',
|
|
19
|
+
r: 'etdf',
|
|
20
|
+
t: 'ryfg',
|
|
21
|
+
y: 'tughzas',
|
|
22
|
+
u: 'yihj',
|
|
23
|
+
i: 'uojk',
|
|
24
|
+
o: 'ipkl',
|
|
25
|
+
p: 'ol',
|
|
26
|
+
a: 'qwszy',
|
|
27
|
+
s: 'awedxzy',
|
|
28
|
+
d: 'serfcx',
|
|
29
|
+
f: 'drtgvc',
|
|
30
|
+
g: 'ftyhbv',
|
|
31
|
+
h: 'gyujnb',
|
|
32
|
+
j: 'huikmn',
|
|
33
|
+
k: 'jiolm',
|
|
34
|
+
l: 'kop',
|
|
35
|
+
z: 'asxtuh',
|
|
36
|
+
x: 'zsdc',
|
|
37
|
+
c: 'xdfv',
|
|
38
|
+
v: 'cfgb',
|
|
39
|
+
b: 'vghn',
|
|
40
|
+
n: 'bhjm',
|
|
41
|
+
m: 'njk',
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* The one-slip misspellings of a (folded) word: a neighbouring key instead of
|
|
45
|
+
* a letter, a letter dropped, a letter doubled, two adjacent letters swapped.
|
|
46
|
+
* De-duplicated, and never the word itself.
|
|
47
|
+
*/
|
|
48
|
+
export function typoVariants(word, kinds = ['neighbour', 'drop', 'double', 'swap']) {
|
|
49
|
+
const s = Array.from(word);
|
|
50
|
+
const out = new Set();
|
|
51
|
+
for (let i = 0; i < s.length; i++) {
|
|
52
|
+
if (kinds.includes('neighbour')) {
|
|
53
|
+
for (const n of NEIGHBOURS[s[i]] ?? '') {
|
|
54
|
+
out.add([...s.slice(0, i), n, ...s.slice(i + 1)].join(''));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
if (kinds.includes('drop') && s.length > 1) {
|
|
58
|
+
out.add([...s.slice(0, i), ...s.slice(i + 1)].join(''));
|
|
59
|
+
}
|
|
60
|
+
if (kinds.includes('double')) {
|
|
61
|
+
out.add([...s.slice(0, i + 1), s[i], ...s.slice(i + 1)].join(''));
|
|
62
|
+
}
|
|
63
|
+
if (kinds.includes('swap') && i + 1 < s.length && s[i] !== s[i + 1]) {
|
|
64
|
+
out.add([...s.slice(0, i), s[i + 1], s[i], ...s.slice(i + 2)].join(''));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
out.delete(word);
|
|
68
|
+
return [...out];
|
|
69
|
+
}
|
|
70
|
+
/** Run every case through `search` (ids, best first) and score it. */
|
|
71
|
+
export async function evaluate(cases, search) {
|
|
72
|
+
let at1 = 0;
|
|
73
|
+
let at3 = 0;
|
|
74
|
+
let rr = 0;
|
|
75
|
+
let zero = 0;
|
|
76
|
+
const misses = [];
|
|
77
|
+
for (const c of cases) {
|
|
78
|
+
const got = await search(c.query);
|
|
79
|
+
if (got.length === 0) {
|
|
80
|
+
zero++;
|
|
81
|
+
}
|
|
82
|
+
const rank = got.findIndex(id => c.expect.includes(id));
|
|
83
|
+
if (rank === 0) {
|
|
84
|
+
at1++;
|
|
85
|
+
}
|
|
86
|
+
if (rank >= 0 && rank < 3) {
|
|
87
|
+
at3++;
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
misses.push({ query: c.query, expect: c.expect, got: got.slice(0, 5) });
|
|
91
|
+
}
|
|
92
|
+
rr += rank >= 0 ? 1 / (rank + 1) : 0;
|
|
93
|
+
}
|
|
94
|
+
const n = cases.length || 1;
|
|
95
|
+
return {
|
|
96
|
+
cases: cases.length,
|
|
97
|
+
successAt1: at1 / n,
|
|
98
|
+
successAt3: at3 / n,
|
|
99
|
+
mrr: rr / n,
|
|
100
|
+
zeroResults: zero / n,
|
|
101
|
+
misses,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Cases from your own names: each name's words, each misspelled once — but
|
|
106
|
+
* only words long enough to be allowed a typo (`typo.ts`), since a 3-letter
|
|
107
|
+
* word with a wrong letter is a different word.
|
|
108
|
+
*/
|
|
109
|
+
export function typoCases(names, options = {}) {
|
|
110
|
+
const { policy = DEFAULT_TYPO_POLICY, perWord = Infinity, fold } = options;
|
|
111
|
+
const cases = [];
|
|
112
|
+
for (const { id, name } of names) {
|
|
113
|
+
// The search's own words by default: "No-screen" is two words to it, so
|
|
114
|
+
// a slip in "no" must not be generated as "bo-screen".
|
|
115
|
+
const words = fold ? fold(name) : foldWords(name);
|
|
116
|
+
words.forEach((word, i) => {
|
|
117
|
+
if (typoBudget(Array.from(word).length, policy) < 1) {
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
for (const variant of typoVariants(word).slice(0, perWord)) {
|
|
121
|
+
const query = [...words.slice(0, i), variant, ...words.slice(i + 1)].join(' ');
|
|
122
|
+
cases.push({ query, expect: [id] });
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
return cases;
|
|
127
|
+
}
|
package/dist/expand.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type TypoPolicy } from './typo.ts';
|
|
2
|
+
import { type Candidate, type Vocabulary } from './vocabulary.ts';
|
|
3
|
+
export interface ExpandedWord {
|
|
4
|
+
/** The word as typed, folded. */
|
|
5
|
+
typed: string;
|
|
6
|
+
/** The collection's words it could mean, best first (exact and prefixes included). */
|
|
7
|
+
candidates: Candidate[];
|
|
8
|
+
}
|
|
9
|
+
export interface ExpandOptions {
|
|
10
|
+
policy?: TypoPolicy;
|
|
11
|
+
/** Per typed word, at most this many corrected spellings. Default 5. */
|
|
12
|
+
perWord?: number;
|
|
13
|
+
/** The same site spelling rule the vocabulary was built with (`SearchSpec.normalize`). */
|
|
14
|
+
normalize?: (word: string) => string;
|
|
15
|
+
}
|
|
16
|
+
/** Every typed word with the collection's words it could mean. */
|
|
17
|
+
export declare function expandQuery(query: string, vocab: Vocabulary, options?: ExpandOptions): ExpandedWord[];
|
|
18
|
+
/**
|
|
19
|
+
* The words to send to an exact/prefix backend so it forgives typos: each
|
|
20
|
+
* correction that is not simply what was typed. "cqto pots" → ["cato", "post"].
|
|
21
|
+
*/
|
|
22
|
+
export declare function corrections(query: string, vocab: Vocabulary, options?: ExpandOptions): string[];
|
|
23
|
+
/**
|
|
24
|
+
* The closest query whose words all exist, or null when every word already
|
|
25
|
+
* does (or none can be corrected). Prefers fewer typos, then the commoner word.
|
|
26
|
+
*/
|
|
27
|
+
export declare function didYouMean(query: string, vocab: Vocabulary, policy?: TypoPolicy): string | null;
|
package/dist/expand.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typo tolerance for a search that runs somewhere else — a database, an API.
|
|
3
|
+
*
|
|
4
|
+
* Expand each typed word into the collection's words it could mean, and send
|
|
5
|
+
* those along with what was typed. The database then only has to match words
|
|
6
|
+
* exactly or by prefix, which every one can do with an index; the forgiving
|
|
7
|
+
* part happens here, the same way for every backend.
|
|
8
|
+
*
|
|
9
|
+
* And when nothing matched: `didYouMean` names the closest query made of
|
|
10
|
+
* words that exist, to offer as a one-tap fix.
|
|
11
|
+
*/
|
|
12
|
+
import { tokenize, writtenLengths } from "./normalize.js";
|
|
13
|
+
import { DEFAULT_TYPO_POLICY } from "./typo.js";
|
|
14
|
+
import { candidates } from "./vocabulary.js";
|
|
15
|
+
/** Every typed word with the collection's words it could mean. */
|
|
16
|
+
export function expandQuery(query, vocab, options = {}) {
|
|
17
|
+
const { policy = DEFAULT_TYPO_POLICY, perWord = 5, normalize = (w) => w } = options;
|
|
18
|
+
const typed = writtenLengths(query, normalize);
|
|
19
|
+
return [...typed].map(([word, writtenLength]) => ({
|
|
20
|
+
typed: word,
|
|
21
|
+
candidates: candidates(vocab, word, { prefix: true, policy, limit: 100, writtenLength })
|
|
22
|
+
.filter(c => c.typos > 0 || !c.prefix || c.term === word)
|
|
23
|
+
.slice(0, perWord),
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The words to send to an exact/prefix backend so it forgives typos: each
|
|
28
|
+
* correction that is not simply what was typed. "cqto pots" → ["cato", "post"].
|
|
29
|
+
*/
|
|
30
|
+
export function corrections(query, vocab, options) {
|
|
31
|
+
const out = new Set();
|
|
32
|
+
for (const w of expandQuery(query, vocab, options)) {
|
|
33
|
+
for (const c of w.candidates) {
|
|
34
|
+
if (c.typos > 0) {
|
|
35
|
+
out.add(c.term);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
return [...out];
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The closest query whose words all exist, or null when every word already
|
|
43
|
+
* does (or none can be corrected). Prefers fewer typos, then the commoner word.
|
|
44
|
+
*/
|
|
45
|
+
export function didYouMean(query, vocab, policy = DEFAULT_TYPO_POLICY) {
|
|
46
|
+
const typed = tokenize(query).map(t => t.term);
|
|
47
|
+
if (typed.length === 0) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
let changed = false;
|
|
51
|
+
const fixed = typed.map(word => {
|
|
52
|
+
if (vocab.counts.has(word)) {
|
|
53
|
+
return word;
|
|
54
|
+
}
|
|
55
|
+
const best = candidates(vocab, word, { prefix: false, policy, limit: 1 })[0];
|
|
56
|
+
if (best) {
|
|
57
|
+
changed = true;
|
|
58
|
+
return best.term;
|
|
59
|
+
}
|
|
60
|
+
return word;
|
|
61
|
+
});
|
|
62
|
+
return changed ? fixed.join(' ') : null;
|
|
63
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Range } from './engine.ts';
|
|
2
|
+
export interface HighlightPart {
|
|
3
|
+
text: string;
|
|
4
|
+
match: boolean;
|
|
5
|
+
}
|
|
6
|
+
/** Split `text` into marked and unmarked runs by `ranges` (overlaps merged). */
|
|
7
|
+
export declare function highlightRanges(text: string, ranges: readonly Range[]): HighlightPart[];
|
|
8
|
+
/**
|
|
9
|
+
* Mark every word of `text` that starts with a word of `query`, by the typed
|
|
10
|
+
* length. For text that was not part of the search (a snippet fetched later).
|
|
11
|
+
*/
|
|
12
|
+
export declare function highlightQuery(text: string, query: string): HighlightPart[];
|
|
13
|
+
export interface Snippet {
|
|
14
|
+
/** The excerpt, with "…" where text was cut. */
|
|
15
|
+
text: string;
|
|
16
|
+
/** The ranges that fall inside it, re-based onto `text`. */
|
|
17
|
+
ranges: Range[];
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The part of a long text worth showing for a match: a window that starts a
|
|
21
|
+
* little before the first matched word, cut at word boundaries, with "…"
|
|
22
|
+
* where it was cut. Without a match it is the text's beginning. For a
|
|
23
|
+
* description whose third paragraph is what matched.
|
|
24
|
+
*/
|
|
25
|
+
export declare function snippet(text: string, ranges: readonly Range[], { maxLength, lead }?: {
|
|
26
|
+
maxLength?: number;
|
|
27
|
+
lead?: number;
|
|
28
|
+
}): Snippet;
|