@bitbaum/searchkit 0.0.0-stage → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +164 -2
- package/dist/distance.d.ts +24 -0
- package/dist/distance.js +91 -0
- package/dist/engine.d.ts +76 -0
- package/dist/engine.js +350 -0
- package/dist/evaluate.d.ts +46 -0
- package/dist/evaluate.js +127 -0
- package/dist/expand.d.ts +27 -0
- package/dist/expand.js +63 -0
- package/dist/highlight.d.ts +28 -0
- package/dist/highlight.js +95 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/normalize.d.ts +45 -0
- package/dist/normalize.js +103 -0
- package/dist/react/index.d.ts +41 -0
- package/dist/react/index.js +59 -0
- package/dist/typo.d.ts +23 -0
- package/dist/typo.js +22 -0
- package/dist/vocabulary.d.ts +38 -0
- package/dist/vocabulary.js +118 -0
- package/package.json +50 -4
- package/src/distance.ts +93 -0
- package/src/engine.ts +492 -0
- package/src/evaluate.ts +162 -0
- package/src/expand.ts +89 -0
- package/src/highlight.ts +115 -0
- package/src/index.ts +52 -0
- package/src/normalize.ts +127 -0
- package/src/react/index.ts +103 -0
- package/src/typo.ts +31 -0
- package/src/vocabulary.ts +164 -0
package/src/distance.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How far apart two words are, in typos.
|
|
3
|
+
*
|
|
4
|
+
* A typo is one of the four things fingers do: a wrong letter (cqto → cato),
|
|
5
|
+
* a missing letter (cto), an extra letter (caato), or two letters swapped
|
|
6
|
+
* (ctao). Counting a swap as ONE edit — optimal string alignment, the
|
|
7
|
+
* restricted Damerau–Levenshtein distance — matters: plain Levenshtein counts
|
|
8
|
+
* "ctao" as two, which spends a whole short word's budget on one slip.
|
|
9
|
+
*
|
|
10
|
+
* Both functions stop as soon as the answer must exceed `max`, so checking a
|
|
11
|
+
* query word against thousands of vocabulary words stays cheap.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Optimal string alignment distance, or `max + 1` when it is larger than
|
|
16
|
+
* `max`. Strings are compared by code point.
|
|
17
|
+
*/
|
|
18
|
+
export function editDistance(a: string, b: string, max = Infinity): number {
|
|
19
|
+
const s = Array.from(a);
|
|
20
|
+
const t = Array.from(b);
|
|
21
|
+
if (Math.abs(s.length - t.length) > max) {
|
|
22
|
+
return max + 1;
|
|
23
|
+
}
|
|
24
|
+
if (s.length === 0 || t.length === 0) {
|
|
25
|
+
const d = Math.max(s.length, t.length);
|
|
26
|
+
return d > max ? max + 1 : d;
|
|
27
|
+
}
|
|
28
|
+
let prevPrev: number[] = [];
|
|
29
|
+
let prev = Array.from({ length: t.length + 1 }, (_, j) => j);
|
|
30
|
+
for (let i = 1; i <= s.length; i++) {
|
|
31
|
+
const row = [i];
|
|
32
|
+
let rowMin = i;
|
|
33
|
+
for (let j = 1; j <= t.length; j++) {
|
|
34
|
+
const cost = s[i - 1] === t[j - 1] ? 0 : 1;
|
|
35
|
+
let d = Math.min(prev[j]! + 1, row[j - 1]! + 1, prev[j - 1]! + cost);
|
|
36
|
+
if (i > 1 && j > 1 && s[i - 1] === t[j - 2] && s[i - 2] === t[j - 1]) {
|
|
37
|
+
d = Math.min(d, prevPrev[j - 2]! + 1);
|
|
38
|
+
}
|
|
39
|
+
row.push(d);
|
|
40
|
+
if (d < rowMin) {
|
|
41
|
+
rowMin = d;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (rowMin > max) {
|
|
45
|
+
return max + 1;
|
|
46
|
+
}
|
|
47
|
+
prevPrev = prev;
|
|
48
|
+
prev = row;
|
|
49
|
+
}
|
|
50
|
+
const d = prev[t.length]!;
|
|
51
|
+
return d > max ? max + 1 : d;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* How many typos it takes to turn `typed` into the START of `word` — the
|
|
56
|
+
* distance that matters while someone is still typing: "bitcon" is one typo
|
|
57
|
+
* from "bitcoin", and "bitco" is zero typos from it. Returns the smallest
|
|
58
|
+
* distance between `typed` and any prefix of `word`, or `max + 1`.
|
|
59
|
+
*/
|
|
60
|
+
export function prefixDistance(typed: string, word: string, max = Infinity): number {
|
|
61
|
+
const s = Array.from(typed);
|
|
62
|
+
const t = Array.from(word);
|
|
63
|
+
if (s.length === 0) {
|
|
64
|
+
return 0;
|
|
65
|
+
}
|
|
66
|
+
// A prefix longer than typed + max can never be within max; one shorter
|
|
67
|
+
// than typed - max neither. Cap the work at the first typed + max letters.
|
|
68
|
+
const cols = Math.min(t.length, s.length + max);
|
|
69
|
+
let prevPrev: number[] = [];
|
|
70
|
+
let prev = Array.from({ length: cols + 1 }, (_, j) => j);
|
|
71
|
+
for (let i = 1; i <= s.length; i++) {
|
|
72
|
+
const row = [i];
|
|
73
|
+
let rowMin = i;
|
|
74
|
+
for (let j = 1; j <= cols; j++) {
|
|
75
|
+
const cost = s[i - 1] === t[j - 1] ? 0 : 1;
|
|
76
|
+
let d = Math.min(prev[j]! + 1, row[j - 1]! + 1, prev[j - 1]! + cost);
|
|
77
|
+
if (i > 1 && j > 1 && s[i - 1] === t[j - 2] && s[i - 2] === t[j - 1]) {
|
|
78
|
+
d = Math.min(d, prevPrev[j - 2]! + 1);
|
|
79
|
+
}
|
|
80
|
+
row.push(d);
|
|
81
|
+
if (d < rowMin) {
|
|
82
|
+
rowMin = d;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if (rowMin > max) {
|
|
86
|
+
return max + 1;
|
|
87
|
+
}
|
|
88
|
+
prevPrev = prev;
|
|
89
|
+
prev = row;
|
|
90
|
+
}
|
|
91
|
+
const best = Math.min(...prev);
|
|
92
|
+
return best > max ? max + 1 : best;
|
|
93
|
+
}
|
package/src/engine.ts
ADDED
|
@@ -0,0 +1,492 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Search a list: find what was meant, and put it first.
|
|
3
|
+
*
|
|
4
|
+
* Matching is per typed word. Each word may match a word of the record
|
|
5
|
+
* exactly, as its start (the reader is still typing), within the typo budget
|
|
6
|
+
* (`typo.ts`), or through a second spelling indexed with the record (German
|
|
7
|
+
* without umlauts, two words written as one). Every typed word must match;
|
|
8
|
+
* a word that matches nothing anywhere is set aside, and when no record has
|
|
9
|
+
* all the rest, the last words are dropped one at a time — so a query with a
|
|
10
|
+
* word too many, or one too mangled to correct, still lands somewhere.
|
|
11
|
+
*
|
|
12
|
+
* Ranking is a tie-break, not a weighted sum. Sums need tuning and drift:
|
|
13
|
+
* a long description outscores a short exact title. Each rule below only
|
|
14
|
+
* decides between records the rules above it called equal:
|
|
15
|
+
*
|
|
16
|
+
* 1. typos — fewer typos; a wrong FIRST letter counts double here
|
|
17
|
+
* (it still matches — Meilisearch refuses it outright, which
|
|
18
|
+
* loses every name whose first letter was mistyped)
|
|
19
|
+
* 2. words — more of the typed words matched (differs only once the
|
|
20
|
+
* query has been relaxed)
|
|
21
|
+
* 3. attribute — matched in a more important field (the order you list them)
|
|
22
|
+
* 4. exact — the whole field typed exactly; then the whole field
|
|
23
|
+
* matched (typos and prefixes allowed: "lki" is all of
|
|
24
|
+
* "Loki"); then whole words over prefixes
|
|
25
|
+
* 5. proximity — the matched words sit closer together, in order
|
|
26
|
+
* 6. position — the match is nearer the start of the field
|
|
27
|
+
* 7. custom — your own `rank` (popularity, recency), higher first
|
|
28
|
+
* 8. length — the shorter field, which is the more specific match
|
|
29
|
+
*
|
|
30
|
+
* It is Algolia's published default order, adjusted for short titles and
|
|
31
|
+
* names, which is what people search a site for.
|
|
32
|
+
*/
|
|
33
|
+
import { joinedPairs, tokenize, umlautSpelling, writtenLengths, type Token } from './normalize.ts';
|
|
34
|
+
import { DEFAULT_TYPO_POLICY, type TypoPolicy } from './typo.ts';
|
|
35
|
+
import { candidates, createVocabulary, type Vocabulary } from './vocabulary.ts';
|
|
36
|
+
|
|
37
|
+
export interface SearchField<T> {
|
|
38
|
+
/** For your own debugging and for `Hit.matches`. */
|
|
39
|
+
name: string;
|
|
40
|
+
get: (item: T) => string | readonly string[] | null | undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface SearchSpec<T> {
|
|
44
|
+
/** Most important first: a title before a description. */
|
|
45
|
+
fields: readonly SearchField<T>[];
|
|
46
|
+
/** The last tie-break, higher first: popularity, recency. */
|
|
47
|
+
rank?: (item: T) => number;
|
|
48
|
+
typos?: TypoPolicy;
|
|
49
|
+
/**
|
|
50
|
+
* Let typed words match the start of longer words. "all" (default): every
|
|
51
|
+
* word, so "bit pay" finds "Bitcoin payments". "last": only the word being
|
|
52
|
+
* typed. "none": whole words only.
|
|
53
|
+
*/
|
|
54
|
+
prefix?: 'all' | 'last' | 'none';
|
|
55
|
+
/**
|
|
56
|
+
* A site's own spelling rule, applied to every word on both sides after
|
|
57
|
+
* the standard folding: Swiss German written as people type it ("ie" and
|
|
58
|
+
* "i", doubled letters), a product's abbreviations. Keep it to how a word
|
|
59
|
+
* is SPELT; meaning belongs elsewhere.
|
|
60
|
+
*/
|
|
61
|
+
normalize?: (word: string) => string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Where a typed word landed in one field, as offsets into the original text. */
|
|
65
|
+
export interface Range {
|
|
66
|
+
start: number;
|
|
67
|
+
end: number;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface Hit<T> {
|
|
71
|
+
item: T;
|
|
72
|
+
/** Index of the item in the list it was searched from. */
|
|
73
|
+
index: number;
|
|
74
|
+
/** The tie-break values, in rule order. Lower sorts first. */
|
|
75
|
+
score: readonly number[];
|
|
76
|
+
/** How many typed words matched. */
|
|
77
|
+
words: number;
|
|
78
|
+
typos: number;
|
|
79
|
+
/** Matched ranges per field name, for highlighting. */
|
|
80
|
+
matches: Record<string, Range[]>;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export interface SearchOptions {
|
|
84
|
+
limit?: number;
|
|
85
|
+
/**
|
|
86
|
+
* When no record has every typed word, drop the last words until some do.
|
|
87
|
+
* Default true.
|
|
88
|
+
*/
|
|
89
|
+
relax?: boolean;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
interface Posting {
|
|
93
|
+
doc: number;
|
|
94
|
+
field: number;
|
|
95
|
+
/** Word position in the field; a joined pair sits at its first word. */
|
|
96
|
+
position: number;
|
|
97
|
+
/** Original-text range to mark. */
|
|
98
|
+
start: number;
|
|
99
|
+
end: number;
|
|
100
|
+
/** A second spelling, not the word as written. */
|
|
101
|
+
alternate: boolean;
|
|
102
|
+
/** The field is exactly this one word (or this joined pair). */
|
|
103
|
+
whole: boolean;
|
|
104
|
+
/**
|
|
105
|
+
* Two words the text wrote apart, indexed together ("cat to" as "catto").
|
|
106
|
+
* Found, but ranked as one typo: it must not beat a real word one slip
|
|
107
|
+
* away ("catto" → Cato). A camelCase or handle pair ("OrangeCat") is the
|
|
108
|
+
* word as written, and is not this.
|
|
109
|
+
*/
|
|
110
|
+
bridged?: boolean;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
interface Indexed {
|
|
114
|
+
vocab: Vocabulary;
|
|
115
|
+
postings: Map<string, Posting[]>;
|
|
116
|
+
/** Words per field per doc, for "the whole field was typed". */
|
|
117
|
+
fieldTerms: string[][][];
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function fieldTexts<T>(field: SearchField<T>, item: T): string[] {
|
|
121
|
+
const v = field.get(item);
|
|
122
|
+
if (v === null || v === undefined) {
|
|
123
|
+
return [];
|
|
124
|
+
}
|
|
125
|
+
return (typeof v === 'string' ? [v] : [...v]).filter(
|
|
126
|
+
(s): s is string => typeof s === 'string' && s.length > 0
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function buildIndex<T>(items: readonly T[], spec: SearchSpec<T>): Indexed {
|
|
131
|
+
const n = spec.normalize ?? ((w: string) => w);
|
|
132
|
+
const postings = new Map<string, Posting[]>();
|
|
133
|
+
const words: string[] = [];
|
|
134
|
+
const fieldTerms: string[][][] = [];
|
|
135
|
+
const post = (term: string, p: Posting) => {
|
|
136
|
+
const list = postings.get(term);
|
|
137
|
+
if (list) {
|
|
138
|
+
list.push(p);
|
|
139
|
+
} else {
|
|
140
|
+
postings.set(term, [p]);
|
|
141
|
+
}
|
|
142
|
+
words.push(term);
|
|
143
|
+
};
|
|
144
|
+
|
|
145
|
+
items.forEach((item, doc) => {
|
|
146
|
+
const perField: string[][] = [];
|
|
147
|
+
spec.fields.forEach((field, f) => {
|
|
148
|
+
const terms: string[] = [];
|
|
149
|
+
// Several values of one field (tags, aliases) are separate texts; a
|
|
150
|
+
// gap of 1000 positions keeps proximity from bridging them.
|
|
151
|
+
fieldTexts(field, item).forEach((text, part) => {
|
|
152
|
+
const tokens: Token[] = tokenize(text);
|
|
153
|
+
const base = part * 1000;
|
|
154
|
+
const whole = tokens.length === 1;
|
|
155
|
+
for (const t of tokens) {
|
|
156
|
+
const term = n(t.term);
|
|
157
|
+
post(term, {
|
|
158
|
+
doc,
|
|
159
|
+
field: f,
|
|
160
|
+
position: base + t.index,
|
|
161
|
+
start: t.start,
|
|
162
|
+
end: t.end,
|
|
163
|
+
alternate: false,
|
|
164
|
+
whole,
|
|
165
|
+
});
|
|
166
|
+
terms.push(term);
|
|
167
|
+
const spelt = umlautSpelling(text.slice(t.start, t.end));
|
|
168
|
+
const other = spelt ? n(spelt) : null;
|
|
169
|
+
if (other && other !== term) {
|
|
170
|
+
post(other, {
|
|
171
|
+
doc,
|
|
172
|
+
field: f,
|
|
173
|
+
position: base + t.index,
|
|
174
|
+
start: t.start,
|
|
175
|
+
end: t.end,
|
|
176
|
+
alternate: true,
|
|
177
|
+
whole,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
joinedPairs(tokens.map(t => n(t.term))).forEach((term, i) => {
|
|
182
|
+
const a = tokens[i]!;
|
|
183
|
+
const b = tokens[i + 1]!;
|
|
184
|
+
post(term, {
|
|
185
|
+
doc,
|
|
186
|
+
field: f,
|
|
187
|
+
position: base + a.index,
|
|
188
|
+
start: a.start,
|
|
189
|
+
end: b.end,
|
|
190
|
+
alternate: true,
|
|
191
|
+
whole: tokens.length === 2,
|
|
192
|
+
bridged: a.end < b.start,
|
|
193
|
+
});
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
perField.push(terms);
|
|
197
|
+
});
|
|
198
|
+
fieldTerms.push(perField);
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
return { vocab: createVocabulary(words), postings, fieldTerms };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** One typed word's best landing in one document. */
|
|
205
|
+
interface Landing {
|
|
206
|
+
typos: number;
|
|
207
|
+
/** Typos for ranking: a wrong first letter counts twice. */
|
|
208
|
+
rankTypos: number;
|
|
209
|
+
field: number;
|
|
210
|
+
position: number;
|
|
211
|
+
prefix: boolean;
|
|
212
|
+
start: number;
|
|
213
|
+
end: number;
|
|
214
|
+
/** Characters of the original word the typed word covered (prefix marks). */
|
|
215
|
+
markEnd: number;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const better = (a: Landing, b: Landing) =>
|
|
219
|
+
a.rankTypos - b.rankTypos ||
|
|
220
|
+
a.field - b.field ||
|
|
221
|
+
Number(a.prefix) - Number(b.prefix) ||
|
|
222
|
+
a.position - b.position;
|
|
223
|
+
|
|
224
|
+
export interface Searcher<T> {
|
|
225
|
+
search(query: string, options?: SearchOptions): Hit<T>[];
|
|
226
|
+
/** The words of the list, for `didYouMean` and for expanding a query sent elsewhere. */
|
|
227
|
+
readonly vocabulary: Vocabulary;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Index `items` once; search them as often as you like. Rebuild when the list
|
|
232
|
+
* changes — indexing is linear and cheap next to a network round trip.
|
|
233
|
+
*/
|
|
234
|
+
export function createSearch<T>(items: readonly T[], spec: SearchSpec<T>): Searcher<T> {
|
|
235
|
+
const index = buildIndex(items, spec);
|
|
236
|
+
const policy = spec.typos ?? DEFAULT_TYPO_POLICY;
|
|
237
|
+
const prefixMode = spec.prefix ?? 'all';
|
|
238
|
+
|
|
239
|
+
// Letters each typed word had before `normalize`, for the typo budget; set
|
|
240
|
+
// per search (searching is synchronous).
|
|
241
|
+
let written = new Map<string, number>();
|
|
242
|
+
|
|
243
|
+
function landingsFor(typed: string, allowPrefix: boolean): Map<number, Landing> {
|
|
244
|
+
const best = new Map<number, Landing>();
|
|
245
|
+
const options = { prefix: allowPrefix, policy, limit: 200, writtenLength: written.get(typed) };
|
|
246
|
+
for (const c of candidates(index.vocab, typed, options)) {
|
|
247
|
+
for (const p of index.postings.get(c.term) ?? []) {
|
|
248
|
+
const typedLength = Array.from(typed).length;
|
|
249
|
+
const firstSlip = c.typos > 0 && Array.from(c.term)[0] !== Array.from(typed)[0];
|
|
250
|
+
const landing: Landing = {
|
|
251
|
+
typos: c.typos,
|
|
252
|
+
rankTypos: c.typos + (firstSlip ? 1 : 0) + (p.bridged ? 1 : 0),
|
|
253
|
+
field: p.field,
|
|
254
|
+
position: p.position,
|
|
255
|
+
prefix: c.prefix,
|
|
256
|
+
start: p.start,
|
|
257
|
+
end: p.end,
|
|
258
|
+
markEnd: c.prefix && !p.alternate ? Math.min(p.end, p.start + typedLength) : p.end,
|
|
259
|
+
};
|
|
260
|
+
const seen = best.get(p.doc);
|
|
261
|
+
if (!seen || better(landing, seen) < 0) {
|
|
262
|
+
best.set(p.doc, landing);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return best;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function run(allTerms: string[], typingLast: boolean, relax: boolean): Hit<T>[] {
|
|
270
|
+
const allLandings = allTerms.map((term, i) =>
|
|
271
|
+
landingsFor(
|
|
272
|
+
term,
|
|
273
|
+
prefixMode === 'all' || (prefixMode === 'last' && typingLast && i === allTerms.length - 1)
|
|
274
|
+
)
|
|
275
|
+
);
|
|
276
|
+
// A word that matches nothing anywhere ("pen" for "open", too short for a
|
|
277
|
+
// typo) cannot help, so with `relax` it stops being required rather than
|
|
278
|
+
// emptying the whole result.
|
|
279
|
+
const keep = allTerms.map((_, i) => !relax || allLandings[i]!.size > 0);
|
|
280
|
+
const terms = allTerms.filter((_, i) => keep[i]);
|
|
281
|
+
const perTerm = allLandings.filter((_, i) => keep[i]);
|
|
282
|
+
const hits: Hit<T>[] = [];
|
|
283
|
+
const first = perTerm[0];
|
|
284
|
+
if (!first) {
|
|
285
|
+
return hits;
|
|
286
|
+
}
|
|
287
|
+
for (const doc of first.keys()) {
|
|
288
|
+
const landings = perTerm.map(m => m.get(doc));
|
|
289
|
+
if (landings.some(l => !l)) {
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
const ls = landings as Landing[];
|
|
293
|
+
const typos = ls.reduce((s, l) => s + l.typos, 0);
|
|
294
|
+
const rankTypos = ls.reduce((s, l) => s + l.rankTypos, 0);
|
|
295
|
+
const field = Math.min(...ls.map(l => l.field));
|
|
296
|
+
const typedWhole = index.fieldTerms[doc]!.some(
|
|
297
|
+
fieldWords =>
|
|
298
|
+
fieldWords.length === terms.length && fieldWords.every((w, i) => w === terms[i])
|
|
299
|
+
);
|
|
300
|
+
const exactWords = ls.filter(l => !l.prefix && l.typos === 0).length;
|
|
301
|
+
// Every word of the matched field was matched: "lki" is the WHOLE of
|
|
302
|
+
// "Loki", but only the first word of "Loki — Capability Layer". And
|
|
303
|
+
// every typed word landed there: for "gallium ch", "ch" alone covering
|
|
304
|
+
// a title "Chalco" (gallium only in its description) is not the name.
|
|
305
|
+
const fieldLength = index.fieldTerms[doc]![field]!.length;
|
|
306
|
+
const coversField =
|
|
307
|
+
ls.every(l => l.field === field) && new Set(ls.map(l => l.position)).size >= fieldLength;
|
|
308
|
+
let proximity = 0;
|
|
309
|
+
for (let i = 1; i < ls.length; i++) {
|
|
310
|
+
const a = ls[i - 1]!;
|
|
311
|
+
const b = ls[i]!;
|
|
312
|
+
const gap = a.field === b.field ? b.position - a.position : 8;
|
|
313
|
+
// In order and adjacent costs 0; out of order or apart costs more, capped.
|
|
314
|
+
proximity += gap === 1 ? 0 : Math.min(gap > 0 ? gap : Math.abs(gap) + 1, 8);
|
|
315
|
+
}
|
|
316
|
+
const position = Math.min(...ls.filter(l => l.field === field).map(l => l.position % 1000));
|
|
317
|
+
const item = items[doc]!;
|
|
318
|
+
const matches: Record<string, Range[]> = {};
|
|
319
|
+
for (const l of ls) {
|
|
320
|
+
const name = spec.fields[l.field]!.name;
|
|
321
|
+
(matches[name] ??= []).push({ start: l.start, end: l.markEnd });
|
|
322
|
+
}
|
|
323
|
+
hits.push({
|
|
324
|
+
item,
|
|
325
|
+
index: doc,
|
|
326
|
+
words: terms.length,
|
|
327
|
+
typos,
|
|
328
|
+
matches,
|
|
329
|
+
score: [
|
|
330
|
+
rankTypos,
|
|
331
|
+
-terms.length,
|
|
332
|
+
field,
|
|
333
|
+
typedWhole ? -1000 : coversField ? -500 - exactWords : -exactWords,
|
|
334
|
+
proximity,
|
|
335
|
+
position,
|
|
336
|
+
-(spec.rank?.(item) ?? 0),
|
|
337
|
+
fieldLength,
|
|
338
|
+
doc,
|
|
339
|
+
],
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
return hits;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
return {
|
|
346
|
+
vocabulary: index.vocab,
|
|
347
|
+
search(query, options = {}) {
|
|
348
|
+
const { limit = Infinity, relax = true } = options;
|
|
349
|
+
written = writtenLengths(query, spec.normalize);
|
|
350
|
+
const terms = [...written.keys()];
|
|
351
|
+
if (terms.length === 0) {
|
|
352
|
+
return [];
|
|
353
|
+
}
|
|
354
|
+
const typingLast = !/\s$/.test(query);
|
|
355
|
+
let hits = run(terms, typingLast, relax);
|
|
356
|
+
// Written apart, meant together ("orange cat" for "Orangecat"), or the
|
|
357
|
+
// other way round ("orangecat" for "Orange Cat" is indexed as a joined
|
|
358
|
+
// pair already; a joined word the index has never seen is split where
|
|
359
|
+
// both halves are words it has).
|
|
360
|
+
if (hits.length === 0 && terms.length > 1) {
|
|
361
|
+
hits = run(joinAdjacent(terms, index.vocab), typingLast, relax);
|
|
362
|
+
}
|
|
363
|
+
if (hits.length === 0) {
|
|
364
|
+
const split = splitUnknown(terms, index.vocab);
|
|
365
|
+
if (split) {
|
|
366
|
+
hits = run(split, typingLast, relax);
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
// No item has every word: keep as many as some item has, whichever
|
|
370
|
+
// they are. "jon smith" finds both Jon Snow and John Smith; dropping
|
|
371
|
+
// only the LAST word would lose John Smith.
|
|
372
|
+
if (relax) {
|
|
373
|
+
for (let keep = terms.length - 1; hits.length === 0 && keep >= 1; keep--) {
|
|
374
|
+
const best = new Map<number, Hit<T>>();
|
|
375
|
+
for (const subset of combinations(terms.length, keep)) {
|
|
376
|
+
const typing = typingLast && subset[subset.length - 1] === terms.length - 1;
|
|
377
|
+
for (const hit of run(
|
|
378
|
+
subset.map(i => terms[i]!),
|
|
379
|
+
typing,
|
|
380
|
+
relax
|
|
381
|
+
)) {
|
|
382
|
+
const seen = best.get(hit.index);
|
|
383
|
+
if (!seen || compareHits(hit, seen) < 0) {
|
|
384
|
+
best.set(hit.index, hit);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
hits = [...best.values()];
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
return hits.sort(compareHits).slice(0, limit);
|
|
392
|
+
},
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Index sets of size k from 0..n-1, in order; past 8 words only the leading
|
|
398
|
+
* ones (a pasted sentence is not worth 2^n searches).
|
|
399
|
+
*/
|
|
400
|
+
function combinations(n: number, k: number): number[][] {
|
|
401
|
+
if (n > 8) {
|
|
402
|
+
return [Array.from({ length: k }, (_, i) => i)];
|
|
403
|
+
}
|
|
404
|
+
const out: number[][] = [];
|
|
405
|
+
const pick = (start: number, chosen: number[]) => {
|
|
406
|
+
if (chosen.length === k) {
|
|
407
|
+
out.push(chosen);
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
for (let i = start; i <= n - (k - chosen.length); i++) {
|
|
411
|
+
pick(i + 1, [...chosen, i]);
|
|
412
|
+
}
|
|
413
|
+
};
|
|
414
|
+
pick(0, []);
|
|
415
|
+
return out;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
/** Join each adjacent pair the vocabulary knows as one word. */
|
|
419
|
+
function joinAdjacent(terms: string[], vocab: Vocabulary): string[] {
|
|
420
|
+
const out: string[] = [];
|
|
421
|
+
for (let i = 0; i < terms.length; i++) {
|
|
422
|
+
const joined = i + 1 < terms.length ? terms[i]! + terms[i + 1]! : null;
|
|
423
|
+
if (joined && vocab.counts.has(joined)) {
|
|
424
|
+
out.push(joined);
|
|
425
|
+
i++;
|
|
426
|
+
} else {
|
|
427
|
+
out.push(terms[i]!);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return out;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/** Split each unknown word at the first point where both halves are known words. */
|
|
434
|
+
function splitUnknown(terms: string[], vocab: Vocabulary): string[] | null {
|
|
435
|
+
let changed = false;
|
|
436
|
+
const out = terms.flatMap(term => {
|
|
437
|
+
if (vocab.counts.has(term)) {
|
|
438
|
+
return [term];
|
|
439
|
+
}
|
|
440
|
+
const letters = Array.from(term);
|
|
441
|
+
for (let i = 2; i <= letters.length - 2; i++) {
|
|
442
|
+
const a = letters.slice(0, i).join('');
|
|
443
|
+
const b = letters.slice(i).join('');
|
|
444
|
+
if (vocab.counts.has(a) && vocab.counts.has(b)) {
|
|
445
|
+
changed = true;
|
|
446
|
+
return [a, b];
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
return [term];
|
|
450
|
+
});
|
|
451
|
+
return changed ? out : null;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** Sort order for hits: the tie-break values, compared left to right. */
|
|
455
|
+
export function compareHits<T>(a: Hit<T>, b: Hit<T>): number {
|
|
456
|
+
for (let i = 0; i < a.score.length; i++) {
|
|
457
|
+
const d = (a.score[i] ?? 0) - (b.score[i] ?? 0);
|
|
458
|
+
if (d !== 0) {
|
|
459
|
+
return d;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
return 0;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** Search a list once, without keeping an index. */
|
|
466
|
+
export function search<T>(
|
|
467
|
+
items: readonly T[],
|
|
468
|
+
spec: SearchSpec<T>,
|
|
469
|
+
query: string,
|
|
470
|
+
options?: SearchOptions
|
|
471
|
+
): Hit<T>[] {
|
|
472
|
+
return createSearch(items, spec).search(query, options);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/**
|
|
476
|
+
* Rank rows another engine already found — a database, an API — by what was
|
|
477
|
+
* typed. Rows the ranker cannot place (they matched on a field you did not
|
|
478
|
+
* pass, or through stemming) keep their original order after the ranked
|
|
479
|
+
* ones: still results, just later ones.
|
|
480
|
+
*/
|
|
481
|
+
export function rerank<T>(
|
|
482
|
+
rows: readonly T[],
|
|
483
|
+
spec: SearchSpec<T>,
|
|
484
|
+
query: string
|
|
485
|
+
): Array<{ item: T; hit: Hit<T> | null }> {
|
|
486
|
+
const hits = createSearch(rows, spec).search(query);
|
|
487
|
+
const placed = new Set(hits.map(h => h.index));
|
|
488
|
+
return [
|
|
489
|
+
...hits.map(hit => ({ item: hit.item, hit })),
|
|
490
|
+
...rows.filter((_, i) => !placed.has(i)).map(item => ({ item, hit: null })),
|
|
491
|
+
];
|
|
492
|
+
}
|