@seekdb/bm25 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +633 -0
- package/dist/index.cjs +277 -0
- package/dist/index.d.cts +40 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.mjs +255 -0
- package/package.json +48 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// index.ts
|
|
21
|
+
var index_exports = {};
|
|
22
|
+
__export(index_exports, {
|
|
23
|
+
Bm25EmbeddingFunction: () => Bm25EmbeddingFunction
|
|
24
|
+
});
|
|
25
|
+
module.exports = __toCommonJS(index_exports);
|
|
26
|
+
var import_seekdb = require("seekdb");
|
|
27
|
+
var import_snowball_stemmers = require("snowball-stemmers");
|
|
28
|
+
var NAME = "bm25";
|
|
29
|
+
var DEFAULT_K = 1.2;
|
|
30
|
+
var DEFAULT_B = 0.75;
|
|
31
|
+
var DEFAULT_AVG_DOC_LENGTH = 256;
|
|
32
|
+
var DEFAULT_TOKEN_MAX_LENGTH = 40;
|
|
33
|
+
var DEFAULT_MAX_DIMENSION = 5e5;
|
|
34
|
+
var DEFAULT_STOPWORDS = [
|
|
35
|
+
"a",
|
|
36
|
+
"an",
|
|
37
|
+
"and",
|
|
38
|
+
"are",
|
|
39
|
+
"as",
|
|
40
|
+
"at",
|
|
41
|
+
"be",
|
|
42
|
+
"by",
|
|
43
|
+
"for",
|
|
44
|
+
"from",
|
|
45
|
+
"has",
|
|
46
|
+
"he",
|
|
47
|
+
"in",
|
|
48
|
+
"is",
|
|
49
|
+
"it",
|
|
50
|
+
"its",
|
|
51
|
+
"of",
|
|
52
|
+
"on",
|
|
53
|
+
"that",
|
|
54
|
+
"the",
|
|
55
|
+
"to",
|
|
56
|
+
"was",
|
|
57
|
+
"were",
|
|
58
|
+
"will",
|
|
59
|
+
"with"
|
|
60
|
+
];
|
|
61
|
+
var ENGLISH_STEMMER = (0, import_snowball_stemmers.newStemmer)("english");
|
|
62
|
+
var Murmur3AbsHasher = class {
|
|
63
|
+
constructor(seed = 0) {
|
|
64
|
+
this.seed = seed;
|
|
65
|
+
}
|
|
66
|
+
murmur3(key) {
|
|
67
|
+
let h1 = this.seed >>> 0;
|
|
68
|
+
const c1 = 3432918353;
|
|
69
|
+
const c2 = 461845907;
|
|
70
|
+
const bytes = key.length - (key.length & 3);
|
|
71
|
+
let i = 0;
|
|
72
|
+
while (i < bytes) {
|
|
73
|
+
let k12 = key.charCodeAt(i) & 255 | (key.charCodeAt(i + 1) & 255) << 8 | (key.charCodeAt(i + 2) & 255) << 16 | (key.charCodeAt(i + 3) & 255) << 24;
|
|
74
|
+
i += 4;
|
|
75
|
+
k12 = Math.imul(k12, c1);
|
|
76
|
+
k12 = k12 << 15 | k12 >>> 17;
|
|
77
|
+
k12 = Math.imul(k12, c2);
|
|
78
|
+
h1 ^= k12;
|
|
79
|
+
h1 = h1 << 13 | h1 >>> 19;
|
|
80
|
+
h1 = Math.imul(h1, 5) + 3864292196;
|
|
81
|
+
}
|
|
82
|
+
let k1 = 0;
|
|
83
|
+
switch (key.length & 3) {
|
|
84
|
+
case 3:
|
|
85
|
+
k1 ^= (key.charCodeAt(i + 2) & 255) << 16;
|
|
86
|
+
case 2:
|
|
87
|
+
k1 ^= (key.charCodeAt(i + 1) & 255) << 8;
|
|
88
|
+
case 1:
|
|
89
|
+
k1 ^= key.charCodeAt(i) & 255;
|
|
90
|
+
k1 = Math.imul(k1, c1);
|
|
91
|
+
k1 = k1 << 15 | k1 >>> 17;
|
|
92
|
+
k1 = Math.imul(k1, c2);
|
|
93
|
+
h1 ^= k1;
|
|
94
|
+
}
|
|
95
|
+
h1 ^= key.length;
|
|
96
|
+
h1 ^= h1 >>> 16;
|
|
97
|
+
h1 = Math.imul(h1, 2246822507);
|
|
98
|
+
h1 ^= h1 >>> 13;
|
|
99
|
+
h1 = Math.imul(h1, 3266489909);
|
|
100
|
+
h1 ^= h1 >>> 16;
|
|
101
|
+
return h1 >>> 0;
|
|
102
|
+
}
|
|
103
|
+
hash(token) {
|
|
104
|
+
const unsigned = this.murmur3(token);
|
|
105
|
+
const signed = unsigned << 0 | 0;
|
|
106
|
+
return Math.abs(signed);
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
var Bm25Tokenizer = class {
|
|
110
|
+
constructor(stemmer, stopwords, tokenMaxLength) {
|
|
111
|
+
this.stemmer = stemmer;
|
|
112
|
+
this.tokenMaxLength = tokenMaxLength;
|
|
113
|
+
this.stopwords = new Set(
|
|
114
|
+
Array.from(stopwords, (word) => word.toLowerCase())
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
removeNonAlphanumeric(text) {
|
|
118
|
+
return text.replace(/[^\p{L}\p{N}_\s]+/gu, " ");
|
|
119
|
+
}
|
|
120
|
+
simpleTokenize(text) {
|
|
121
|
+
return text.toLowerCase().split(/\s+/u).filter(Boolean);
|
|
122
|
+
}
|
|
123
|
+
tokenize(text) {
|
|
124
|
+
const cleaned = this.removeNonAlphanumeric(text);
|
|
125
|
+
const rawTokens = this.simpleTokenize(cleaned);
|
|
126
|
+
const tokens = [];
|
|
127
|
+
for (const token of rawTokens) {
|
|
128
|
+
if (token.length === 0) continue;
|
|
129
|
+
if (this.stopwords.has(token)) continue;
|
|
130
|
+
if (token.length > this.tokenMaxLength) continue;
|
|
131
|
+
const stemmed = this.stemmer.stem(token).trim();
|
|
132
|
+
if (stemmed.length > 0) {
|
|
133
|
+
tokens.push(stemmed);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return tokens;
|
|
137
|
+
}
|
|
138
|
+
};
|
|
139
|
+
var Bm25EmbeddingFunction = class _Bm25EmbeddingFunction {
|
|
140
|
+
constructor(args = {}) {
|
|
141
|
+
this.name = NAME;
|
|
142
|
+
const {
|
|
143
|
+
k = DEFAULT_K,
|
|
144
|
+
b = DEFAULT_B,
|
|
145
|
+
avgDocLength = DEFAULT_AVG_DOC_LENGTH,
|
|
146
|
+
tokenMaxLength = DEFAULT_TOKEN_MAX_LENGTH,
|
|
147
|
+
maxDimension = DEFAULT_MAX_DIMENSION,
|
|
148
|
+
stopwords
|
|
149
|
+
} = args;
|
|
150
|
+
if (!Number.isFinite(k) || k <= 0) {
|
|
151
|
+
throw new import_seekdb.SeekdbValueError("k must be a positive finite number");
|
|
152
|
+
}
|
|
153
|
+
if (!Number.isFinite(b) || b < 0 || b > 1) {
|
|
154
|
+
throw new import_seekdb.SeekdbValueError("b must be a finite number in [0, 1]");
|
|
155
|
+
}
|
|
156
|
+
if (!Number.isFinite(avgDocLength) || avgDocLength <= 0) {
|
|
157
|
+
throw new import_seekdb.SeekdbValueError(
|
|
158
|
+
"avgDocLength must be a positive finite number"
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
if (!Number.isInteger(tokenMaxLength) || tokenMaxLength <= 0) {
|
|
162
|
+
throw new import_seekdb.SeekdbValueError("tokenMaxLength must be a positive integer");
|
|
163
|
+
}
|
|
164
|
+
if (!Number.isInteger(maxDimension) || maxDimension <= 0) {
|
|
165
|
+
throw new import_seekdb.SeekdbValueError("maxDimension must be a positive integer");
|
|
166
|
+
}
|
|
167
|
+
this.k = k;
|
|
168
|
+
this.b = b;
|
|
169
|
+
this.avgDocLength = avgDocLength;
|
|
170
|
+
this.tokenMaxLength = tokenMaxLength;
|
|
171
|
+
this.maxDimension = maxDimension;
|
|
172
|
+
this.customStopwords = stopwords ? [...stopwords] : void 0;
|
|
173
|
+
const stopwordList = this.customStopwords ?? [...DEFAULT_STOPWORDS];
|
|
174
|
+
this.tokenizer = new Bm25Tokenizer(
|
|
175
|
+
ENGLISH_STEMMER,
|
|
176
|
+
stopwordList,
|
|
177
|
+
tokenMaxLength
|
|
178
|
+
);
|
|
179
|
+
this.hasher = new Murmur3AbsHasher();
|
|
180
|
+
}
|
|
181
|
+
encode(text) {
|
|
182
|
+
const tokenList = this.tokenizer.tokenize(text);
|
|
183
|
+
if (tokenList.length === 0) {
|
|
184
|
+
return {};
|
|
185
|
+
}
|
|
186
|
+
const docLen = tokenList.length;
|
|
187
|
+
const counts = /* @__PURE__ */ new Map();
|
|
188
|
+
for (const token of tokenList) {
|
|
189
|
+
const tokenId = this.hasher.hash(token) % this.maxDimension;
|
|
190
|
+
counts.set(tokenId, (counts.get(tokenId) ?? 0) + 1);
|
|
191
|
+
}
|
|
192
|
+
const sparseVector = {};
|
|
193
|
+
for (const tokenId of Array.from(counts.keys()).sort((a, b) => a - b)) {
|
|
194
|
+
const tf = counts.get(tokenId);
|
|
195
|
+
const denominator = tf + this.k * (1 - this.b + this.b * docLen / this.avgDocLength);
|
|
196
|
+
sparseVector[tokenId] = tf * (this.k + 1) / denominator;
|
|
197
|
+
}
|
|
198
|
+
return sparseVector;
|
|
199
|
+
}
|
|
200
|
+
async generate(texts) {
|
|
201
|
+
if (!Array.isArray(texts)) {
|
|
202
|
+
throw new import_seekdb.SeekdbValueError("texts must be an array of strings");
|
|
203
|
+
}
|
|
204
|
+
if (texts.length === 0) {
|
|
205
|
+
return [];
|
|
206
|
+
}
|
|
207
|
+
return texts.map((text) => this.encode(text));
|
|
208
|
+
}
|
|
209
|
+
async generateForQueries(texts) {
|
|
210
|
+
return this.generate(texts);
|
|
211
|
+
}
|
|
212
|
+
getConfig() {
|
|
213
|
+
const config = {
|
|
214
|
+
k: this.k,
|
|
215
|
+
b: this.b,
|
|
216
|
+
avg_doc_length: this.avgDocLength,
|
|
217
|
+
token_max_length: this.tokenMaxLength,
|
|
218
|
+
max_dimension: this.maxDimension
|
|
219
|
+
};
|
|
220
|
+
if (this.customStopwords) {
|
|
221
|
+
config.stopwords = [...this.customStopwords];
|
|
222
|
+
}
|
|
223
|
+
return config;
|
|
224
|
+
}
|
|
225
|
+
validateConfigUpdate(newConfig) {
|
|
226
|
+
const mutableKeys = /* @__PURE__ */ new Set([
|
|
227
|
+
"k",
|
|
228
|
+
"b",
|
|
229
|
+
"avg_doc_length",
|
|
230
|
+
"token_max_length",
|
|
231
|
+
"max_dimension",
|
|
232
|
+
"stopwords"
|
|
233
|
+
]);
|
|
234
|
+
for (const key of Object.keys(newConfig)) {
|
|
235
|
+
if (!mutableKeys.has(key)) {
|
|
236
|
+
throw new import_seekdb.SeekdbValueError(
|
|
237
|
+
`Updating '${key}' is not supported for ${NAME}`
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
static validateConfig(config) {
|
|
243
|
+
if (config.k !== void 0 && (!Number.isFinite(config.k) || config.k <= 0)) {
|
|
244
|
+
throw new import_seekdb.SeekdbValueError("k must be a positive finite number");
|
|
245
|
+
}
|
|
246
|
+
if (config.b !== void 0 && (!Number.isFinite(config.b) || config.b < 0 || config.b > 1)) {
|
|
247
|
+
throw new import_seekdb.SeekdbValueError("b must be a finite number in [0, 1]");
|
|
248
|
+
}
|
|
249
|
+
if (config.avg_doc_length !== void 0 && (!Number.isFinite(config.avg_doc_length) || config.avg_doc_length <= 0)) {
|
|
250
|
+
throw new import_seekdb.SeekdbValueError(
|
|
251
|
+
"avg_doc_length must be a positive finite number"
|
|
252
|
+
);
|
|
253
|
+
}
|
|
254
|
+
if (config.token_max_length !== void 0 && (!Number.isInteger(config.token_max_length) || config.token_max_length <= 0)) {
|
|
255
|
+
throw new import_seekdb.SeekdbValueError("token_max_length must be a positive integer");
|
|
256
|
+
}
|
|
257
|
+
if (config.max_dimension !== void 0 && (!Number.isInteger(config.max_dimension) || config.max_dimension <= 0)) {
|
|
258
|
+
throw new import_seekdb.SeekdbValueError("max_dimension must be a positive integer");
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
static buildFromConfig(config) {
|
|
262
|
+
_Bm25EmbeddingFunction.validateConfig(config);
|
|
263
|
+
return new _Bm25EmbeddingFunction({
|
|
264
|
+
k: config.k,
|
|
265
|
+
b: config.b,
|
|
266
|
+
avgDocLength: config.avg_doc_length,
|
|
267
|
+
tokenMaxLength: config.token_max_length,
|
|
268
|
+
maxDimension: config.max_dimension,
|
|
269
|
+
stopwords: config.stopwords
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
(0, import_seekdb.registerSparseEmbeddingFunction)(NAME, Bm25EmbeddingFunction);
|
|
274
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
275
|
+
0 && (module.exports = {
|
|
276
|
+
Bm25EmbeddingFunction
|
|
277
|
+
});
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { EmbeddingConfig, SparseEmbeddingFunction, SparseVector } from 'seekdb';
|
|
2
|
+
|
|
3
|
+
interface Bm25EmbeddingArgs extends EmbeddingConfig {
|
|
4
|
+
k?: number;
|
|
5
|
+
b?: number;
|
|
6
|
+
avgDocLength?: number;
|
|
7
|
+
tokenMaxLength?: number;
|
|
8
|
+
/** Max sparse dimension index. Default 500000. */
|
|
9
|
+
maxDimension?: number;
|
|
10
|
+
stopwords?: string[];
|
|
11
|
+
}
|
|
12
|
+
interface Bm25EmbeddingConfig extends EmbeddingConfig {
|
|
13
|
+
k?: number;
|
|
14
|
+
b?: number;
|
|
15
|
+
avg_doc_length?: number;
|
|
16
|
+
token_max_length?: number;
|
|
17
|
+
max_dimension?: number;
|
|
18
|
+
stopwords?: string[];
|
|
19
|
+
}
|
|
20
|
+
declare class Bm25EmbeddingFunction implements SparseEmbeddingFunction {
|
|
21
|
+
readonly name = "bm25";
|
|
22
|
+
private readonly tokenizer;
|
|
23
|
+
private readonly hasher;
|
|
24
|
+
private readonly k;
|
|
25
|
+
private readonly b;
|
|
26
|
+
private readonly avgDocLength;
|
|
27
|
+
private readonly tokenMaxLength;
|
|
28
|
+
private readonly maxDimension;
|
|
29
|
+
private readonly customStopwords?;
|
|
30
|
+
constructor(args?: Bm25EmbeddingArgs);
|
|
31
|
+
private encode;
|
|
32
|
+
generate(texts: string[]): Promise<SparseVector[]>;
|
|
33
|
+
generateForQueries(texts: string[]): Promise<SparseVector[]>;
|
|
34
|
+
getConfig(): Bm25EmbeddingConfig;
|
|
35
|
+
validateConfigUpdate(newConfig: Record<string, unknown>): void;
|
|
36
|
+
static validateConfig(config: Bm25EmbeddingConfig): void;
|
|
37
|
+
static buildFromConfig(config: Bm25EmbeddingConfig): Bm25EmbeddingFunction;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export { type Bm25EmbeddingArgs, type Bm25EmbeddingConfig, Bm25EmbeddingFunction };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { EmbeddingConfig, SparseEmbeddingFunction, SparseVector } from 'seekdb';
|
|
2
|
+
|
|
3
|
+
interface Bm25EmbeddingArgs extends EmbeddingConfig {
|
|
4
|
+
k?: number;
|
|
5
|
+
b?: number;
|
|
6
|
+
avgDocLength?: number;
|
|
7
|
+
tokenMaxLength?: number;
|
|
8
|
+
/** Max sparse dimension index. Default 500000. */
|
|
9
|
+
maxDimension?: number;
|
|
10
|
+
stopwords?: string[];
|
|
11
|
+
}
|
|
12
|
+
interface Bm25EmbeddingConfig extends EmbeddingConfig {
|
|
13
|
+
k?: number;
|
|
14
|
+
b?: number;
|
|
15
|
+
avg_doc_length?: number;
|
|
16
|
+
token_max_length?: number;
|
|
17
|
+
max_dimension?: number;
|
|
18
|
+
stopwords?: string[];
|
|
19
|
+
}
|
|
20
|
+
declare class Bm25EmbeddingFunction implements SparseEmbeddingFunction {
|
|
21
|
+
readonly name = "bm25";
|
|
22
|
+
private readonly tokenizer;
|
|
23
|
+
private readonly hasher;
|
|
24
|
+
private readonly k;
|
|
25
|
+
private readonly b;
|
|
26
|
+
private readonly avgDocLength;
|
|
27
|
+
private readonly tokenMaxLength;
|
|
28
|
+
private readonly maxDimension;
|
|
29
|
+
private readonly customStopwords?;
|
|
30
|
+
constructor(args?: Bm25EmbeddingArgs);
|
|
31
|
+
private encode;
|
|
32
|
+
generate(texts: string[]): Promise<SparseVector[]>;
|
|
33
|
+
generateForQueries(texts: string[]): Promise<SparseVector[]>;
|
|
34
|
+
getConfig(): Bm25EmbeddingConfig;
|
|
35
|
+
validateConfigUpdate(newConfig: Record<string, unknown>): void;
|
|
36
|
+
static validateConfig(config: Bm25EmbeddingConfig): void;
|
|
37
|
+
static buildFromConfig(config: Bm25EmbeddingConfig): Bm25EmbeddingFunction;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export { type Bm25EmbeddingArgs, type Bm25EmbeddingConfig, Bm25EmbeddingFunction };
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
// index.ts
|
|
2
|
+
import {
|
|
3
|
+
registerSparseEmbeddingFunction,
|
|
4
|
+
SeekdbValueError
|
|
5
|
+
} from "seekdb";
|
|
6
|
+
import { newStemmer } from "snowball-stemmers";
|
|
7
|
+
var NAME = "bm25";
|
|
8
|
+
var DEFAULT_K = 1.2;
|
|
9
|
+
var DEFAULT_B = 0.75;
|
|
10
|
+
var DEFAULT_AVG_DOC_LENGTH = 256;
|
|
11
|
+
var DEFAULT_TOKEN_MAX_LENGTH = 40;
|
|
12
|
+
var DEFAULT_MAX_DIMENSION = 5e5;
|
|
13
|
+
var DEFAULT_STOPWORDS = [
|
|
14
|
+
"a",
|
|
15
|
+
"an",
|
|
16
|
+
"and",
|
|
17
|
+
"are",
|
|
18
|
+
"as",
|
|
19
|
+
"at",
|
|
20
|
+
"be",
|
|
21
|
+
"by",
|
|
22
|
+
"for",
|
|
23
|
+
"from",
|
|
24
|
+
"has",
|
|
25
|
+
"he",
|
|
26
|
+
"in",
|
|
27
|
+
"is",
|
|
28
|
+
"it",
|
|
29
|
+
"its",
|
|
30
|
+
"of",
|
|
31
|
+
"on",
|
|
32
|
+
"that",
|
|
33
|
+
"the",
|
|
34
|
+
"to",
|
|
35
|
+
"was",
|
|
36
|
+
"were",
|
|
37
|
+
"will",
|
|
38
|
+
"with"
|
|
39
|
+
];
|
|
40
|
+
var ENGLISH_STEMMER = newStemmer("english");
|
|
41
|
+
var Murmur3AbsHasher = class {
|
|
42
|
+
constructor(seed = 0) {
|
|
43
|
+
this.seed = seed;
|
|
44
|
+
}
|
|
45
|
+
murmur3(key) {
|
|
46
|
+
let h1 = this.seed >>> 0;
|
|
47
|
+
const c1 = 3432918353;
|
|
48
|
+
const c2 = 461845907;
|
|
49
|
+
const bytes = key.length - (key.length & 3);
|
|
50
|
+
let i = 0;
|
|
51
|
+
while (i < bytes) {
|
|
52
|
+
let k12 = key.charCodeAt(i) & 255 | (key.charCodeAt(i + 1) & 255) << 8 | (key.charCodeAt(i + 2) & 255) << 16 | (key.charCodeAt(i + 3) & 255) << 24;
|
|
53
|
+
i += 4;
|
|
54
|
+
k12 = Math.imul(k12, c1);
|
|
55
|
+
k12 = k12 << 15 | k12 >>> 17;
|
|
56
|
+
k12 = Math.imul(k12, c2);
|
|
57
|
+
h1 ^= k12;
|
|
58
|
+
h1 = h1 << 13 | h1 >>> 19;
|
|
59
|
+
h1 = Math.imul(h1, 5) + 3864292196;
|
|
60
|
+
}
|
|
61
|
+
let k1 = 0;
|
|
62
|
+
switch (key.length & 3) {
|
|
63
|
+
case 3:
|
|
64
|
+
k1 ^= (key.charCodeAt(i + 2) & 255) << 16;
|
|
65
|
+
case 2:
|
|
66
|
+
k1 ^= (key.charCodeAt(i + 1) & 255) << 8;
|
|
67
|
+
case 1:
|
|
68
|
+
k1 ^= key.charCodeAt(i) & 255;
|
|
69
|
+
k1 = Math.imul(k1, c1);
|
|
70
|
+
k1 = k1 << 15 | k1 >>> 17;
|
|
71
|
+
k1 = Math.imul(k1, c2);
|
|
72
|
+
h1 ^= k1;
|
|
73
|
+
}
|
|
74
|
+
h1 ^= key.length;
|
|
75
|
+
h1 ^= h1 >>> 16;
|
|
76
|
+
h1 = Math.imul(h1, 2246822507);
|
|
77
|
+
h1 ^= h1 >>> 13;
|
|
78
|
+
h1 = Math.imul(h1, 3266489909);
|
|
79
|
+
h1 ^= h1 >>> 16;
|
|
80
|
+
return h1 >>> 0;
|
|
81
|
+
}
|
|
82
|
+
hash(token) {
|
|
83
|
+
const unsigned = this.murmur3(token);
|
|
84
|
+
const signed = unsigned << 0 | 0;
|
|
85
|
+
return Math.abs(signed);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
var Bm25Tokenizer = class {
|
|
89
|
+
constructor(stemmer, stopwords, tokenMaxLength) {
|
|
90
|
+
this.stemmer = stemmer;
|
|
91
|
+
this.tokenMaxLength = tokenMaxLength;
|
|
92
|
+
this.stopwords = new Set(
|
|
93
|
+
Array.from(stopwords, (word) => word.toLowerCase())
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
removeNonAlphanumeric(text) {
|
|
97
|
+
return text.replace(/[^\p{L}\p{N}_\s]+/gu, " ");
|
|
98
|
+
}
|
|
99
|
+
simpleTokenize(text) {
|
|
100
|
+
return text.toLowerCase().split(/\s+/u).filter(Boolean);
|
|
101
|
+
}
|
|
102
|
+
tokenize(text) {
|
|
103
|
+
const cleaned = this.removeNonAlphanumeric(text);
|
|
104
|
+
const rawTokens = this.simpleTokenize(cleaned);
|
|
105
|
+
const tokens = [];
|
|
106
|
+
for (const token of rawTokens) {
|
|
107
|
+
if (token.length === 0) continue;
|
|
108
|
+
if (this.stopwords.has(token)) continue;
|
|
109
|
+
if (token.length > this.tokenMaxLength) continue;
|
|
110
|
+
const stemmed = this.stemmer.stem(token).trim();
|
|
111
|
+
if (stemmed.length > 0) {
|
|
112
|
+
tokens.push(stemmed);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return tokens;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
var Bm25EmbeddingFunction = class _Bm25EmbeddingFunction {
|
|
119
|
+
constructor(args = {}) {
|
|
120
|
+
this.name = NAME;
|
|
121
|
+
const {
|
|
122
|
+
k = DEFAULT_K,
|
|
123
|
+
b = DEFAULT_B,
|
|
124
|
+
avgDocLength = DEFAULT_AVG_DOC_LENGTH,
|
|
125
|
+
tokenMaxLength = DEFAULT_TOKEN_MAX_LENGTH,
|
|
126
|
+
maxDimension = DEFAULT_MAX_DIMENSION,
|
|
127
|
+
stopwords
|
|
128
|
+
} = args;
|
|
129
|
+
if (!Number.isFinite(k) || k <= 0) {
|
|
130
|
+
throw new SeekdbValueError("k must be a positive finite number");
|
|
131
|
+
}
|
|
132
|
+
if (!Number.isFinite(b) || b < 0 || b > 1) {
|
|
133
|
+
throw new SeekdbValueError("b must be a finite number in [0, 1]");
|
|
134
|
+
}
|
|
135
|
+
if (!Number.isFinite(avgDocLength) || avgDocLength <= 0) {
|
|
136
|
+
throw new SeekdbValueError(
|
|
137
|
+
"avgDocLength must be a positive finite number"
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
if (!Number.isInteger(tokenMaxLength) || tokenMaxLength <= 0) {
|
|
141
|
+
throw new SeekdbValueError("tokenMaxLength must be a positive integer");
|
|
142
|
+
}
|
|
143
|
+
if (!Number.isInteger(maxDimension) || maxDimension <= 0) {
|
|
144
|
+
throw new SeekdbValueError("maxDimension must be a positive integer");
|
|
145
|
+
}
|
|
146
|
+
this.k = k;
|
|
147
|
+
this.b = b;
|
|
148
|
+
this.avgDocLength = avgDocLength;
|
|
149
|
+
this.tokenMaxLength = tokenMaxLength;
|
|
150
|
+
this.maxDimension = maxDimension;
|
|
151
|
+
this.customStopwords = stopwords ? [...stopwords] : void 0;
|
|
152
|
+
const stopwordList = this.customStopwords ?? [...DEFAULT_STOPWORDS];
|
|
153
|
+
this.tokenizer = new Bm25Tokenizer(
|
|
154
|
+
ENGLISH_STEMMER,
|
|
155
|
+
stopwordList,
|
|
156
|
+
tokenMaxLength
|
|
157
|
+
);
|
|
158
|
+
this.hasher = new Murmur3AbsHasher();
|
|
159
|
+
}
|
|
160
|
+
encode(text) {
|
|
161
|
+
const tokenList = this.tokenizer.tokenize(text);
|
|
162
|
+
if (tokenList.length === 0) {
|
|
163
|
+
return {};
|
|
164
|
+
}
|
|
165
|
+
const docLen = tokenList.length;
|
|
166
|
+
const counts = /* @__PURE__ */ new Map();
|
|
167
|
+
for (const token of tokenList) {
|
|
168
|
+
const tokenId = this.hasher.hash(token) % this.maxDimension;
|
|
169
|
+
counts.set(tokenId, (counts.get(tokenId) ?? 0) + 1);
|
|
170
|
+
}
|
|
171
|
+
const sparseVector = {};
|
|
172
|
+
for (const tokenId of Array.from(counts.keys()).sort((a, b) => a - b)) {
|
|
173
|
+
const tf = counts.get(tokenId);
|
|
174
|
+
const denominator = tf + this.k * (1 - this.b + this.b * docLen / this.avgDocLength);
|
|
175
|
+
sparseVector[tokenId] = tf * (this.k + 1) / denominator;
|
|
176
|
+
}
|
|
177
|
+
return sparseVector;
|
|
178
|
+
}
|
|
179
|
+
async generate(texts) {
|
|
180
|
+
if (!Array.isArray(texts)) {
|
|
181
|
+
throw new SeekdbValueError("texts must be an array of strings");
|
|
182
|
+
}
|
|
183
|
+
if (texts.length === 0) {
|
|
184
|
+
return [];
|
|
185
|
+
}
|
|
186
|
+
return texts.map((text) => this.encode(text));
|
|
187
|
+
}
|
|
188
|
+
async generateForQueries(texts) {
|
|
189
|
+
return this.generate(texts);
|
|
190
|
+
}
|
|
191
|
+
getConfig() {
|
|
192
|
+
const config = {
|
|
193
|
+
k: this.k,
|
|
194
|
+
b: this.b,
|
|
195
|
+
avg_doc_length: this.avgDocLength,
|
|
196
|
+
token_max_length: this.tokenMaxLength,
|
|
197
|
+
max_dimension: this.maxDimension
|
|
198
|
+
};
|
|
199
|
+
if (this.customStopwords) {
|
|
200
|
+
config.stopwords = [...this.customStopwords];
|
|
201
|
+
}
|
|
202
|
+
return config;
|
|
203
|
+
}
|
|
204
|
+
validateConfigUpdate(newConfig) {
|
|
205
|
+
const mutableKeys = /* @__PURE__ */ new Set([
|
|
206
|
+
"k",
|
|
207
|
+
"b",
|
|
208
|
+
"avg_doc_length",
|
|
209
|
+
"token_max_length",
|
|
210
|
+
"max_dimension",
|
|
211
|
+
"stopwords"
|
|
212
|
+
]);
|
|
213
|
+
for (const key of Object.keys(newConfig)) {
|
|
214
|
+
if (!mutableKeys.has(key)) {
|
|
215
|
+
throw new SeekdbValueError(
|
|
216
|
+
`Updating '${key}' is not supported for ${NAME}`
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
static validateConfig(config) {
|
|
222
|
+
if (config.k !== void 0 && (!Number.isFinite(config.k) || config.k <= 0)) {
|
|
223
|
+
throw new SeekdbValueError("k must be a positive finite number");
|
|
224
|
+
}
|
|
225
|
+
if (config.b !== void 0 && (!Number.isFinite(config.b) || config.b < 0 || config.b > 1)) {
|
|
226
|
+
throw new SeekdbValueError("b must be a finite number in [0, 1]");
|
|
227
|
+
}
|
|
228
|
+
if (config.avg_doc_length !== void 0 && (!Number.isFinite(config.avg_doc_length) || config.avg_doc_length <= 0)) {
|
|
229
|
+
throw new SeekdbValueError(
|
|
230
|
+
"avg_doc_length must be a positive finite number"
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
if (config.token_max_length !== void 0 && (!Number.isInteger(config.token_max_length) || config.token_max_length <= 0)) {
|
|
234
|
+
throw new SeekdbValueError("token_max_length must be a positive integer");
|
|
235
|
+
}
|
|
236
|
+
if (config.max_dimension !== void 0 && (!Number.isInteger(config.max_dimension) || config.max_dimension <= 0)) {
|
|
237
|
+
throw new SeekdbValueError("max_dimension must be a positive integer");
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
static buildFromConfig(config) {
|
|
241
|
+
_Bm25EmbeddingFunction.validateConfig(config);
|
|
242
|
+
return new _Bm25EmbeddingFunction({
|
|
243
|
+
k: config.k,
|
|
244
|
+
b: config.b,
|
|
245
|
+
avgDocLength: config.avg_doc_length,
|
|
246
|
+
tokenMaxLength: config.token_max_length,
|
|
247
|
+
maxDimension: config.max_dimension,
|
|
248
|
+
stopwords: config.stopwords
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
registerSparseEmbeddingFunction(NAME, Bm25EmbeddingFunction);
|
|
253
|
+
export {
|
|
254
|
+
Bm25EmbeddingFunction
|
|
255
|
+
};
|
package/package.json
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@seekdb/bm25",
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"main": "dist/index.cjs",
|
|
6
|
+
"module": "dist/index.mjs",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.mjs",
|
|
12
|
+
"require": "./dist/index.cjs"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"dist"
|
|
17
|
+
],
|
|
18
|
+
"peerDependencies": {
|
|
19
|
+
"@seekdb/common": "1.2.0",
|
|
20
|
+
"seekdb": "1.2.0"
|
|
21
|
+
},
|
|
22
|
+
"devDependencies": {
|
|
23
|
+
"@seekdb/common": "1.2.0",
|
|
24
|
+
"seekdb": "1.2.0"
|
|
25
|
+
},
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@types/snowball-stemmers": "^0.6.2",
|
|
28
|
+
"snowball-stemmers": "^0.6.0"
|
|
29
|
+
},
|
|
30
|
+
"publishConfig": {
|
|
31
|
+
"registry": "https://registry.npmjs.org",
|
|
32
|
+
"access": "public"
|
|
33
|
+
},
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "https://github.com/oceanbase/seekdb-js.git",
|
|
37
|
+
"directory": "packages/embeddings/bm25"
|
|
38
|
+
},
|
|
39
|
+
"homepage": "https://github.com/oceanbase/seekdb-js#readme",
|
|
40
|
+
"bugs": {
|
|
41
|
+
"url": "https://github.com/oceanbase/seekdb-js/issues"
|
|
42
|
+
},
|
|
43
|
+
"scripts": {
|
|
44
|
+
"build": "tsup",
|
|
45
|
+
"dev": "tsup --watch",
|
|
46
|
+
"test": "vitest"
|
|
47
|
+
}
|
|
48
|
+
}
|