@bitbaum/searchkit 0.0.0-stage → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,93 @@
1
+ /**
2
+ * How far apart two words are, in typos.
3
+ *
4
+ * A typo is one of the four things fingers do: a wrong letter (cqto → cato),
5
+ * a missing letter (cto), an extra letter (caato), or two letters swapped
6
+ * (ctao). Counting a swap as ONE edit — optimal string alignment, the
7
+ * restricted Damerau–Levenshtein distance — matters: plain Levenshtein counts
8
+ * "ctao" as two, which spends a whole short word's budget on one slip.
9
+ *
10
+ * Both functions stop as soon as the answer must exceed `max`, so checking a
11
+ * query word against thousands of vocabulary words stays cheap.
12
+ */
13
+
14
+ /**
15
+ * Optimal string alignment distance, or `max + 1` when it is larger than
16
+ * `max`. Strings are compared by code point.
17
+ */
18
+ export function editDistance(a: string, b: string, max = Infinity): number {
19
+ const s = Array.from(a);
20
+ const t = Array.from(b);
21
+ if (Math.abs(s.length - t.length) > max) {
22
+ return max + 1;
23
+ }
24
+ if (s.length === 0 || t.length === 0) {
25
+ const d = Math.max(s.length, t.length);
26
+ return d > max ? max + 1 : d;
27
+ }
28
+ let prevPrev: number[] = [];
29
+ let prev = Array.from({ length: t.length + 1 }, (_, j) => j);
30
+ for (let i = 1; i <= s.length; i++) {
31
+ const row = [i];
32
+ let rowMin = i;
33
+ for (let j = 1; j <= t.length; j++) {
34
+ const cost = s[i - 1] === t[j - 1] ? 0 : 1;
35
+ let d = Math.min(prev[j]! + 1, row[j - 1]! + 1, prev[j - 1]! + cost);
36
+ if (i > 1 && j > 1 && s[i - 1] === t[j - 2] && s[i - 2] === t[j - 1]) {
37
+ d = Math.min(d, prevPrev[j - 2]! + 1);
38
+ }
39
+ row.push(d);
40
+ if (d < rowMin) {
41
+ rowMin = d;
42
+ }
43
+ }
44
+ if (rowMin > max) {
45
+ return max + 1;
46
+ }
47
+ prevPrev = prev;
48
+ prev = row;
49
+ }
50
+ const d = prev[t.length]!;
51
+ return d > max ? max + 1 : d;
52
+ }
53
+
54
+ /**
55
+ * How many typos it takes to turn `typed` into the START of `word` — the
56
+ * distance that matters while someone is still typing: "bitcon" is one typo
57
+ * from "bitcoin", and "bitco" is zero typos from it. Returns the smallest
58
+ * distance between `typed` and any prefix of `word`, or `max + 1`.
59
+ */
60
+ export function prefixDistance(typed: string, word: string, max = Infinity): number {
61
+ const s = Array.from(typed);
62
+ const t = Array.from(word);
63
+ if (s.length === 0) {
64
+ return 0;
65
+ }
66
+ // A prefix longer than typed + max can never be within max; one shorter
67
+ // than typed - max neither. Cap the work at the first typed + max letters.
68
+ const cols = Math.min(t.length, s.length + max);
69
+ let prevPrev: number[] = [];
70
+ let prev = Array.from({ length: cols + 1 }, (_, j) => j);
71
+ for (let i = 1; i <= s.length; i++) {
72
+ const row = [i];
73
+ let rowMin = i;
74
+ for (let j = 1; j <= cols; j++) {
75
+ const cost = s[i - 1] === t[j - 1] ? 0 : 1;
76
+ let d = Math.min(prev[j]! + 1, row[j - 1]! + 1, prev[j - 1]! + cost);
77
+ if (i > 1 && j > 1 && s[i - 1] === t[j - 2] && s[i - 2] === t[j - 1]) {
78
+ d = Math.min(d, prevPrev[j - 2]! + 1);
79
+ }
80
+ row.push(d);
81
+ if (d < rowMin) {
82
+ rowMin = d;
83
+ }
84
+ }
85
+ if (rowMin > max) {
86
+ return max + 1;
87
+ }
88
+ prevPrev = prev;
89
+ prev = row;
90
+ }
91
+ const best = Math.min(...prev);
92
+ return best > max ? max + 1 : best;
93
+ }
package/src/engine.ts ADDED
@@ -0,0 +1,492 @@
1
+ /**
2
+ * Search a list: find what was meant, and put it first.
3
+ *
4
+ * Matching is per typed word. Each word may match a word of the record
5
+ * exactly, as its start (the reader is still typing), within the typo budget
6
+ * (`typo.ts`), or through a second spelling indexed with the record (German
7
+ * without umlauts, two words written as one). Every typed word must match;
8
+ * a word that matches nothing anywhere is set aside, and when no record has
9
+ * all the rest, the last words are dropped one at a time — so a query with a
10
+ * word too many, or one too mangled to correct, still lands somewhere.
11
+ *
12
+ * Ranking is a tie-break, not a weighted sum. Sums need tuning and drift:
13
+ * a long description outscores a short exact title. Each rule below only
14
+ * decides between records the rules above it called equal:
15
+ *
16
+ * 1. typos — fewer typos; a wrong FIRST letter counts double here
17
+ * (it still matches — Meilisearch refuses it outright, which
18
+ * loses every name whose first letter was mistyped)
19
+ * 2. words — more of the typed words matched (differs only once the
20
+ * query has been relaxed)
21
+ * 3. attribute — matched in a more important field (the order you list them)
22
+ * 4. exact — the whole field typed exactly; then the whole field
23
+ * matched (typos and prefixes allowed: "lki" is all of
24
+ * "Loki"); then whole words over prefixes
25
+ * 5. proximity — the matched words sit closer together, in order
26
+ * 6. position — the match is nearer the start of the field
27
+ * 7. custom — your own `rank` (popularity, recency), higher first
28
+ * 8. length — the shorter field, which is the more specific match
29
+ *
30
+ * It is Algolia's published default order, adjusted for short titles and
31
+ * names, which is what people search a site for.
32
+ */
33
+ import { joinedPairs, tokenize, umlautSpelling, writtenLengths, type Token } from './normalize.ts';
34
+ import { DEFAULT_TYPO_POLICY, type TypoPolicy } from './typo.ts';
35
+ import { candidates, createVocabulary, type Vocabulary } from './vocabulary.ts';
36
+
37
+ export interface SearchField<T> {
38
+ /** For your own debugging and for `Hit.matches`. */
39
+ name: string;
40
+ get: (item: T) => string | readonly string[] | null | undefined;
41
+ }
42
+
43
+ export interface SearchSpec<T> {
44
+ /** Most important first: a title before a description. */
45
+ fields: readonly SearchField<T>[];
46
+ /** The last tie-break, higher first: popularity, recency. */
47
+ rank?: (item: T) => number;
48
+ typos?: TypoPolicy;
49
+ /**
50
+ * Let typed words match the start of longer words. "all" (default): every
51
+ * word, so "bit pay" finds "Bitcoin payments". "last": only the word being
52
+ * typed. "none": whole words only.
53
+ */
54
+ prefix?: 'all' | 'last' | 'none';
55
+ /**
56
+ * A site's own spelling rule, applied to every word on both sides after
57
+ * the standard folding: Swiss German written as people type it ("ie" and
58
+ * "i", doubled letters), a product's abbreviations. Keep it to how a word
59
+ * is SPELT; meaning belongs elsewhere.
60
+ */
61
+ normalize?: (word: string) => string;
62
+ }
63
+
64
+ /** Where a typed word landed in one field, as offsets into the original text. */
65
+ export interface Range {
66
+ start: number;
67
+ end: number;
68
+ }
69
+
70
+ export interface Hit<T> {
71
+ item: T;
72
+ /** Index of the item in the list it was searched from. */
73
+ index: number;
74
+ /** The tie-break values, in rule order. Lower sorts first. */
75
+ score: readonly number[];
76
+ /** How many typed words matched. */
77
+ words: number;
78
+ typos: number;
79
+ /** Matched ranges per field name, for highlighting. */
80
+ matches: Record<string, Range[]>;
81
+ }
82
+
83
+ export interface SearchOptions {
84
+ limit?: number;
85
+ /**
86
+ * When no record has every typed word, drop the last words until some do.
87
+ * Default true.
88
+ */
89
+ relax?: boolean;
90
+ }
91
+
92
+ interface Posting {
93
+ doc: number;
94
+ field: number;
95
+ /** Word position in the field; a joined pair sits at its first word. */
96
+ position: number;
97
+ /** Original-text range to mark. */
98
+ start: number;
99
+ end: number;
100
+ /** A second spelling, not the word as written. */
101
+ alternate: boolean;
102
+ /** The field is exactly this one word (or this joined pair). */
103
+ whole: boolean;
104
+ /**
105
+ * Two words the text wrote apart, indexed together ("cat to" as "catto").
106
+ * Found, but ranked as one typo: it must not beat a real word one slip
107
+ * away ("catto" → Cato). A camelCase or handle pair ("OrangeCat") is the
108
+ * word as written, and is not this.
109
+ */
110
+ bridged?: boolean;
111
+ }
112
+
113
+ interface Indexed {
114
+ vocab: Vocabulary;
115
+ postings: Map<string, Posting[]>;
116
+ /** Words per field per doc, for "the whole field was typed". */
117
+ fieldTerms: string[][][];
118
+ }
119
+
120
+ function fieldTexts<T>(field: SearchField<T>, item: T): string[] {
121
+ const v = field.get(item);
122
+ if (v === null || v === undefined) {
123
+ return [];
124
+ }
125
+ return (typeof v === 'string' ? [v] : [...v]).filter(
126
+ (s): s is string => typeof s === 'string' && s.length > 0
127
+ );
128
+ }
129
+
130
+ function buildIndex<T>(items: readonly T[], spec: SearchSpec<T>): Indexed {
131
+ const n = spec.normalize ?? ((w: string) => w);
132
+ const postings = new Map<string, Posting[]>();
133
+ const words: string[] = [];
134
+ const fieldTerms: string[][][] = [];
135
+ const post = (term: string, p: Posting) => {
136
+ const list = postings.get(term);
137
+ if (list) {
138
+ list.push(p);
139
+ } else {
140
+ postings.set(term, [p]);
141
+ }
142
+ words.push(term);
143
+ };
144
+
145
+ items.forEach((item, doc) => {
146
+ const perField: string[][] = [];
147
+ spec.fields.forEach((field, f) => {
148
+ const terms: string[] = [];
149
+ // Several values of one field (tags, aliases) are separate texts; a
150
+ // gap of 1000 positions keeps proximity from bridging them.
151
+ fieldTexts(field, item).forEach((text, part) => {
152
+ const tokens: Token[] = tokenize(text);
153
+ const base = part * 1000;
154
+ const whole = tokens.length === 1;
155
+ for (const t of tokens) {
156
+ const term = n(t.term);
157
+ post(term, {
158
+ doc,
159
+ field: f,
160
+ position: base + t.index,
161
+ start: t.start,
162
+ end: t.end,
163
+ alternate: false,
164
+ whole,
165
+ });
166
+ terms.push(term);
167
+ const spelt = umlautSpelling(text.slice(t.start, t.end));
168
+ const other = spelt ? n(spelt) : null;
169
+ if (other && other !== term) {
170
+ post(other, {
171
+ doc,
172
+ field: f,
173
+ position: base + t.index,
174
+ start: t.start,
175
+ end: t.end,
176
+ alternate: true,
177
+ whole,
178
+ });
179
+ }
180
+ }
181
+ joinedPairs(tokens.map(t => n(t.term))).forEach((term, i) => {
182
+ const a = tokens[i]!;
183
+ const b = tokens[i + 1]!;
184
+ post(term, {
185
+ doc,
186
+ field: f,
187
+ position: base + a.index,
188
+ start: a.start,
189
+ end: b.end,
190
+ alternate: true,
191
+ whole: tokens.length === 2,
192
+ bridged: a.end < b.start,
193
+ });
194
+ });
195
+ });
196
+ perField.push(terms);
197
+ });
198
+ fieldTerms.push(perField);
199
+ });
200
+
201
+ return { vocab: createVocabulary(words), postings, fieldTerms };
202
+ }
203
+
204
+ /** One typed word's best landing in one document. */
205
+ interface Landing {
206
+ typos: number;
207
+ /** Typos for ranking: a wrong first letter counts twice. */
208
+ rankTypos: number;
209
+ field: number;
210
+ position: number;
211
+ prefix: boolean;
212
+ start: number;
213
+ end: number;
214
+ /** Characters of the original word the typed word covered (prefix marks). */
215
+ markEnd: number;
216
+ }
217
+
218
+ const better = (a: Landing, b: Landing) =>
219
+ a.rankTypos - b.rankTypos ||
220
+ a.field - b.field ||
221
+ Number(a.prefix) - Number(b.prefix) ||
222
+ a.position - b.position;
223
+
224
+ export interface Searcher<T> {
225
+ search(query: string, options?: SearchOptions): Hit<T>[];
226
+ /** The words of the list, for `didYouMean` and for expanding a query sent elsewhere. */
227
+ readonly vocabulary: Vocabulary;
228
+ }
229
+
230
+ /**
231
+ * Index `items` once; search them as often as you like. Rebuild when the list
232
+ * changes — indexing is linear and cheap next to a network round trip.
233
+ */
234
+ export function createSearch<T>(items: readonly T[], spec: SearchSpec<T>): Searcher<T> {
235
+ const index = buildIndex(items, spec);
236
+ const policy = spec.typos ?? DEFAULT_TYPO_POLICY;
237
+ const prefixMode = spec.prefix ?? 'all';
238
+
239
+ // Letters each typed word had before `normalize`, for the typo budget; set
240
+ // per search (searching is synchronous).
241
+ let written = new Map<string, number>();
242
+
243
+ function landingsFor(typed: string, allowPrefix: boolean): Map<number, Landing> {
244
+ const best = new Map<number, Landing>();
245
+ const options = { prefix: allowPrefix, policy, limit: 200, writtenLength: written.get(typed) };
246
+ for (const c of candidates(index.vocab, typed, options)) {
247
+ for (const p of index.postings.get(c.term) ?? []) {
248
+ const typedLength = Array.from(typed).length;
249
+ const firstSlip = c.typos > 0 && Array.from(c.term)[0] !== Array.from(typed)[0];
250
+ const landing: Landing = {
251
+ typos: c.typos,
252
+ rankTypos: c.typos + (firstSlip ? 1 : 0) + (p.bridged ? 1 : 0),
253
+ field: p.field,
254
+ position: p.position,
255
+ prefix: c.prefix,
256
+ start: p.start,
257
+ end: p.end,
258
+ markEnd: c.prefix && !p.alternate ? Math.min(p.end, p.start + typedLength) : p.end,
259
+ };
260
+ const seen = best.get(p.doc);
261
+ if (!seen || better(landing, seen) < 0) {
262
+ best.set(p.doc, landing);
263
+ }
264
+ }
265
+ }
266
+ return best;
267
+ }
268
+
269
+ function run(allTerms: string[], typingLast: boolean, relax: boolean): Hit<T>[] {
270
+ const allLandings = allTerms.map((term, i) =>
271
+ landingsFor(
272
+ term,
273
+ prefixMode === 'all' || (prefixMode === 'last' && typingLast && i === allTerms.length - 1)
274
+ )
275
+ );
276
+ // A word that matches nothing anywhere ("pen" for "open", too short for a
277
+ // typo) cannot help, so with `relax` it stops being required rather than
278
+ // emptying the whole result.
279
+ const keep = allTerms.map((_, i) => !relax || allLandings[i]!.size > 0);
280
+ const terms = allTerms.filter((_, i) => keep[i]);
281
+ const perTerm = allLandings.filter((_, i) => keep[i]);
282
+ const hits: Hit<T>[] = [];
283
+ const first = perTerm[0];
284
+ if (!first) {
285
+ return hits;
286
+ }
287
+ for (const doc of first.keys()) {
288
+ const landings = perTerm.map(m => m.get(doc));
289
+ if (landings.some(l => !l)) {
290
+ continue;
291
+ }
292
+ const ls = landings as Landing[];
293
+ const typos = ls.reduce((s, l) => s + l.typos, 0);
294
+ const rankTypos = ls.reduce((s, l) => s + l.rankTypos, 0);
295
+ const field = Math.min(...ls.map(l => l.field));
296
+ const typedWhole = index.fieldTerms[doc]!.some(
297
+ fieldWords =>
298
+ fieldWords.length === terms.length && fieldWords.every((w, i) => w === terms[i])
299
+ );
300
+ const exactWords = ls.filter(l => !l.prefix && l.typos === 0).length;
301
+ // Every word of the matched field was matched: "lki" is the WHOLE of
302
+ // "Loki", but only the first word of "Loki — Capability Layer". And
303
+ // every typed word landed there: for "gallium ch", "ch" alone covering
304
+ // a title "Chalco" (gallium only in its description) is not the name.
305
+ const fieldLength = index.fieldTerms[doc]![field]!.length;
306
+ const coversField =
307
+ ls.every(l => l.field === field) && new Set(ls.map(l => l.position)).size >= fieldLength;
308
+ let proximity = 0;
309
+ for (let i = 1; i < ls.length; i++) {
310
+ const a = ls[i - 1]!;
311
+ const b = ls[i]!;
312
+ const gap = a.field === b.field ? b.position - a.position : 8;
313
+ // In order and adjacent costs 0; out of order or apart costs more, capped.
314
+ proximity += gap === 1 ? 0 : Math.min(gap > 0 ? gap : Math.abs(gap) + 1, 8);
315
+ }
316
+ const position = Math.min(...ls.filter(l => l.field === field).map(l => l.position % 1000));
317
+ const item = items[doc]!;
318
+ const matches: Record<string, Range[]> = {};
319
+ for (const l of ls) {
320
+ const name = spec.fields[l.field]!.name;
321
+ (matches[name] ??= []).push({ start: l.start, end: l.markEnd });
322
+ }
323
+ hits.push({
324
+ item,
325
+ index: doc,
326
+ words: terms.length,
327
+ typos,
328
+ matches,
329
+ score: [
330
+ rankTypos,
331
+ -terms.length,
332
+ field,
333
+ typedWhole ? -1000 : coversField ? -500 - exactWords : -exactWords,
334
+ proximity,
335
+ position,
336
+ -(spec.rank?.(item) ?? 0),
337
+ fieldLength,
338
+ doc,
339
+ ],
340
+ });
341
+ }
342
+ return hits;
343
+ }
344
+
345
+ return {
346
+ vocabulary: index.vocab,
347
+ search(query, options = {}) {
348
+ const { limit = Infinity, relax = true } = options;
349
+ written = writtenLengths(query, spec.normalize);
350
+ const terms = [...written.keys()];
351
+ if (terms.length === 0) {
352
+ return [];
353
+ }
354
+ const typingLast = !/\s$/.test(query);
355
+ let hits = run(terms, typingLast, relax);
356
+ // Written apart, meant together ("orange cat" for "Orangecat"), or the
357
+ // other way round ("orangecat" for "Orange Cat" is indexed as a joined
358
+ // pair already; a joined word the index has never seen is split where
359
+ // both halves are words it has).
360
+ if (hits.length === 0 && terms.length > 1) {
361
+ hits = run(joinAdjacent(terms, index.vocab), typingLast, relax);
362
+ }
363
+ if (hits.length === 0) {
364
+ const split = splitUnknown(terms, index.vocab);
365
+ if (split) {
366
+ hits = run(split, typingLast, relax);
367
+ }
368
+ }
369
+ // No item has every word: keep as many as some item has, whichever
370
+ // they are. "jon smith" finds both Jon Snow and John Smith; dropping
371
+ // only the LAST word would lose John Smith.
372
+ if (relax) {
373
+ for (let keep = terms.length - 1; hits.length === 0 && keep >= 1; keep--) {
374
+ const best = new Map<number, Hit<T>>();
375
+ for (const subset of combinations(terms.length, keep)) {
376
+ const typing = typingLast && subset[subset.length - 1] === terms.length - 1;
377
+ for (const hit of run(
378
+ subset.map(i => terms[i]!),
379
+ typing,
380
+ relax
381
+ )) {
382
+ const seen = best.get(hit.index);
383
+ if (!seen || compareHits(hit, seen) < 0) {
384
+ best.set(hit.index, hit);
385
+ }
386
+ }
387
+ }
388
+ hits = [...best.values()];
389
+ }
390
+ }
391
+ return hits.sort(compareHits).slice(0, limit);
392
+ },
393
+ };
394
+ }
395
+
396
+ /**
397
+ * Index sets of size k from 0..n-1, in order; past 8 words only the leading
398
+ * ones (a pasted sentence is not worth 2^n searches).
399
+ */
400
+ function combinations(n: number, k: number): number[][] {
401
+ if (n > 8) {
402
+ return [Array.from({ length: k }, (_, i) => i)];
403
+ }
404
+ const out: number[][] = [];
405
+ const pick = (start: number, chosen: number[]) => {
406
+ if (chosen.length === k) {
407
+ out.push(chosen);
408
+ return;
409
+ }
410
+ for (let i = start; i <= n - (k - chosen.length); i++) {
411
+ pick(i + 1, [...chosen, i]);
412
+ }
413
+ };
414
+ pick(0, []);
415
+ return out;
416
+ }
417
+
418
+ /** Join each adjacent pair the vocabulary knows as one word. */
419
+ function joinAdjacent(terms: string[], vocab: Vocabulary): string[] {
420
+ const out: string[] = [];
421
+ for (let i = 0; i < terms.length; i++) {
422
+ const joined = i + 1 < terms.length ? terms[i]! + terms[i + 1]! : null;
423
+ if (joined && vocab.counts.has(joined)) {
424
+ out.push(joined);
425
+ i++;
426
+ } else {
427
+ out.push(terms[i]!);
428
+ }
429
+ }
430
+ return out;
431
+ }
432
+
433
+ /** Split each unknown word at the first point where both halves are known words. */
434
+ function splitUnknown(terms: string[], vocab: Vocabulary): string[] | null {
435
+ let changed = false;
436
+ const out = terms.flatMap(term => {
437
+ if (vocab.counts.has(term)) {
438
+ return [term];
439
+ }
440
+ const letters = Array.from(term);
441
+ for (let i = 2; i <= letters.length - 2; i++) {
442
+ const a = letters.slice(0, i).join('');
443
+ const b = letters.slice(i).join('');
444
+ if (vocab.counts.has(a) && vocab.counts.has(b)) {
445
+ changed = true;
446
+ return [a, b];
447
+ }
448
+ }
449
+ return [term];
450
+ });
451
+ return changed ? out : null;
452
+ }
453
+
454
+ /** Sort order for hits: the tie-break values, compared left to right. */
455
+ export function compareHits<T>(a: Hit<T>, b: Hit<T>): number {
456
+ for (let i = 0; i < a.score.length; i++) {
457
+ const d = (a.score[i] ?? 0) - (b.score[i] ?? 0);
458
+ if (d !== 0) {
459
+ return d;
460
+ }
461
+ }
462
+ return 0;
463
+ }
464
+
465
+ /** Search a list once, without keeping an index. */
466
+ export function search<T>(
467
+ items: readonly T[],
468
+ spec: SearchSpec<T>,
469
+ query: string,
470
+ options?: SearchOptions
471
+ ): Hit<T>[] {
472
+ return createSearch(items, spec).search(query, options);
473
+ }
474
+
475
+ /**
476
+ * Rank rows another engine already found — a database, an API — by what was
477
+ * typed. Rows the ranker cannot place (they matched on a field you did not
478
+ * pass, or through stemming) keep their original order after the ranked
479
+ * ones: still results, just later ones.
480
+ */
481
+ export function rerank<T>(
482
+ rows: readonly T[],
483
+ spec: SearchSpec<T>,
484
+ query: string
485
+ ): Array<{ item: T; hit: Hit<T> | null }> {
486
+ const hits = createSearch(rows, spec).search(query);
487
+ const placed = new Set(hits.map(h => h.index));
488
+ return [
489
+ ...hits.map(hit => ({ item: hit.item, hit })),
490
+ ...rows.filter((_, i) => !placed.has(i)).map(item => ({ item, hit: null })),
491
+ ];
492
+ }