@o-lang/semantic-doc-search 1.0.23 → 1.0.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/config.json +25 -0
  2. package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/onnx/model_quantized.onnx +0 -0
  3. package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/tokenizer.json +30686 -0
  4. package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/tokenizer_config.json +15 -0
  5. package/embeddings.json +1 -514
  6. package/package.json +2 -1
  7. package/src/embeddings/local.js +77 -86
  8. package/src/index.js +76 -36
  9. package/src/test-doc-search.js +13 -0
  10. package/test-doc-search-batch.js +36 -0
  11. package/test-doc-search.js +22 -0
  12. package/test-single-doc.js +32 -0
package/.cache/embeddings/Xenova/all-MiniLM-L6-v2/tokenizer_config.json ADDED
@@ -0,0 +1,15 @@
1
+ {
2
+ "clean_up_tokenization_spaces": true,
3
+ "cls_token": "[CLS]",
4
+ "do_basic_tokenize": true,
5
+ "do_lower_case": true,
6
+ "mask_token": "[MASK]",
7
+ "model_max_length": 512,
8
+ "never_split": null,
9
+ "pad_token": "[PAD]",
10
+ "sep_token": "[SEP]",
11
+ "strip_accents": null,
12
+ "tokenize_chinese_chars": true,
13
+ "tokenizer_class": "BertTokenizer",
14
+ "unk_token": "[UNK]"
15
+ }
package/embeddings.json CHANGED
@@ -1,516 +1,3 @@
1
1
  {
2
- "sample1.txt::chunk::0": [
3
- 0.21568627450980393,
4
- 0.0784313725490196,
5
- 0.0196078431372549,
6
- 0.796078431372549,
7
- 0.10980392156862745,
8
- 0.4627450980392157,
9
- 0.9490196078431372,
10
- 0.24313725490196078,
11
- 0.8117647058823529,
12
- 0.6,
13
- 0.06274509803921569,
14
- 0.1411764705882353,
15
- 0.1450980392156863,
16
- 0.3176470588235294,
17
- 0.1568627450980392,
18
- 0.8666666666666667,
19
- 0.3568627450980392,
20
- 0.2823529411764706,
21
- 0.24705882352941178,
22
- 0.5294117647058824,
23
- 0.5019607843137255,
24
- 0.8313725490196079,
25
- 0.7058823529411765,
26
- 0.6980392156862745,
27
- 0.9764705882352941,
28
- 0.9450980392156862,
29
- 0.43137254901960786,
30
- 0.00784313725490196,
31
- 0.33725490196078434,
32
- 0.9333333333333333,
33
- 0.1568627450980392,
34
- 0.792156862745098,
35
- 0.21568627450980393,
36
- 0.0784313725490196,
37
- 0.0196078431372549,
38
- 0.796078431372549,
39
- 0.10980392156862745,
40
- 0.4627450980392157,
41
- 0.9490196078431372,
42
- 0.24313725490196078,
43
- 0.8117647058823529,
44
- 0.6,
45
- 0.06274509803921569,
46
- 0.1411764705882353,
47
- 0.1450980392156863,
48
- 0.3176470588235294,
49
- 0.1568627450980392,
50
- 0.8666666666666667,
51
- 0.3568627450980392,
52
- 0.2823529411764706,
53
- 0.24705882352941178,
54
- 0.5294117647058824,
55
- 0.5019607843137255,
56
- 0.8313725490196079,
57
- 0.7058823529411765,
58
- 0.6980392156862745,
59
- 0.9764705882352941,
60
- 0.9450980392156862,
61
- 0.43137254901960786,
62
- 0.00784313725490196,
63
- 0.33725490196078434,
64
- 0.9333333333333333,
65
- 0.1568627450980392,
66
- 0.792156862745098,
67
- 0.21568627450980393,
68
- 0.0784313725490196,
69
- 0.0196078431372549,
70
- 0.796078431372549,
71
- 0.10980392156862745,
72
- 0.4627450980392157,
73
- 0.9490196078431372,
74
- 0.24313725490196078,
75
- 0.8117647058823529,
76
- 0.6,
77
- 0.06274509803921569,
78
- 0.1411764705882353,
79
- 0.1450980392156863,
80
- 0.3176470588235294,
81
- 0.1568627450980392,
82
- 0.8666666666666667,
83
- 0.3568627450980392,
84
- 0.2823529411764706,
85
- 0.24705882352941178,
86
- 0.5294117647058824,
87
- 0.5019607843137255,
88
- 0.8313725490196079,
89
- 0.7058823529411765,
90
- 0.6980392156862745,
91
- 0.9764705882352941,
92
- 0.9450980392156862,
93
- 0.43137254901960786,
94
- 0.00784313725490196,
95
- 0.33725490196078434,
96
- 0.9333333333333333,
97
- 0.1568627450980392,
98
- 0.792156862745098,
99
- 0.21568627450980393,
100
- 0.0784313725490196,
101
- 0.0196078431372549,
102
- 0.796078431372549,
103
- 0.10980392156862745,
104
- 0.4627450980392157,
105
- 0.9490196078431372,
106
- 0.24313725490196078,
107
- 0.8117647058823529,
108
- 0.6,
109
- 0.06274509803921569,
110
- 0.1411764705882353,
111
- 0.1450980392156863,
112
- 0.3176470588235294,
113
- 0.1568627450980392,
114
- 0.8666666666666667,
115
- 0.3568627450980392,
116
- 0.2823529411764706,
117
- 0.24705882352941178,
118
- 0.5294117647058824,
119
- 0.5019607843137255,
120
- 0.8313725490196079,
121
- 0.7058823529411765,
122
- 0.6980392156862745,
123
- 0.9764705882352941,
124
- 0.9450980392156862,
125
- 0.43137254901960786,
126
- 0.00784313725490196,
127
- 0.33725490196078434,
128
- 0.9333333333333333,
129
- 0.1568627450980392,
130
- 0.792156862745098,
131
- 0.21568627450980393,
132
- 0.0784313725490196,
133
- 0.0196078431372549,
134
- 0.796078431372549,
135
- 0.10980392156862745,
136
- 0.4627450980392157,
137
- 0.9490196078431372,
138
- 0.24313725490196078,
139
- 0.8117647058823529,
140
- 0.6,
141
- 0.06274509803921569,
142
- 0.1411764705882353,
143
- 0.1450980392156863,
144
- 0.3176470588235294,
145
- 0.1568627450980392,
146
- 0.8666666666666667,
147
- 0.3568627450980392,
148
- 0.2823529411764706,
149
- 0.24705882352941178,
150
- 0.5294117647058824,
151
- 0.5019607843137255,
152
- 0.8313725490196079,
153
- 0.7058823529411765,
154
- 0.6980392156862745,
155
- 0.9764705882352941,
156
- 0.9450980392156862,
157
- 0.43137254901960786,
158
- 0.00784313725490196,
159
- 0.33725490196078434,
160
- 0.9333333333333333,
161
- 0.1568627450980392,
162
- 0.792156862745098,
163
- 0.21568627450980393,
164
- 0.0784313725490196,
165
- 0.0196078431372549,
166
- 0.796078431372549,
167
- 0.10980392156862745,
168
- 0.4627450980392157,
169
- 0.9490196078431372,
170
- 0.24313725490196078,
171
- 0.8117647058823529,
172
- 0.6,
173
- 0.06274509803921569,
174
- 0.1411764705882353,
175
- 0.1450980392156863,
176
- 0.3176470588235294,
177
- 0.1568627450980392,
178
- 0.8666666666666667,
179
- 0.3568627450980392,
180
- 0.2823529411764706,
181
- 0.24705882352941178,
182
- 0.5294117647058824,
183
- 0.5019607843137255,
184
- 0.8313725490196079,
185
- 0.7058823529411765,
186
- 0.6980392156862745,
187
- 0.9764705882352941,
188
- 0.9450980392156862,
189
- 0.43137254901960786,
190
- 0.00784313725490196,
191
- 0.33725490196078434,
192
- 0.9333333333333333,
193
- 0.1568627450980392,
194
- 0.792156862745098,
195
- 0.21568627450980393,
196
- 0.0784313725490196,
197
- 0.0196078431372549,
198
- 0.796078431372549,
199
- 0.10980392156862745,
200
- 0.4627450980392157,
201
- 0.9490196078431372,
202
- 0.24313725490196078,
203
- 0.8117647058823529,
204
- 0.6,
205
- 0.06274509803921569,
206
- 0.1411764705882353,
207
- 0.1450980392156863,
208
- 0.3176470588235294,
209
- 0.1568627450980392,
210
- 0.8666666666666667,
211
- 0.3568627450980392,
212
- 0.2823529411764706,
213
- 0.24705882352941178,
214
- 0.5294117647058824,
215
- 0.5019607843137255,
216
- 0.8313725490196079,
217
- 0.7058823529411765,
218
- 0.6980392156862745,
219
- 0.9764705882352941,
220
- 0.9450980392156862,
221
- 0.43137254901960786,
222
- 0.00784313725490196,
223
- 0.33725490196078434,
224
- 0.9333333333333333,
225
- 0.1568627450980392,
226
- 0.792156862745098,
227
- 0.21568627450980393,
228
- 0.0784313725490196,
229
- 0.0196078431372549,
230
- 0.796078431372549,
231
- 0.10980392156862745,
232
- 0.4627450980392157,
233
- 0.9490196078431372,
234
- 0.24313725490196078,
235
- 0.8117647058823529,
236
- 0.6,
237
- 0.06274509803921569,
238
- 0.1411764705882353,
239
- 0.1450980392156863,
240
- 0.3176470588235294,
241
- 0.1568627450980392,
242
- 0.8666666666666667,
243
- 0.3568627450980392,
244
- 0.2823529411764706,
245
- 0.24705882352941178,
246
- 0.5294117647058824,
247
- 0.5019607843137255,
248
- 0.8313725490196079,
249
- 0.7058823529411765,
250
- 0.6980392156862745,
251
- 0.9764705882352941,
252
- 0.9450980392156862,
253
- 0.43137254901960786,
254
- 0.00784313725490196,
255
- 0.33725490196078434,
256
- 0.9333333333333333,
257
- 0.1568627450980392,
258
- 0.792156862745098,
259
- 0.21568627450980393,
260
- 0.0784313725490196,
261
- 0.0196078431372549,
262
- 0.796078431372549,
263
- 0.10980392156862745,
264
- 0.4627450980392157,
265
- 0.9490196078431372,
266
- 0.24313725490196078,
267
- 0.8117647058823529,
268
- 0.6,
269
- 0.06274509803921569,
270
- 0.1411764705882353,
271
- 0.1450980392156863,
272
- 0.3176470588235294,
273
- 0.1568627450980392,
274
- 0.8666666666666667,
275
- 0.3568627450980392,
276
- 0.2823529411764706,
277
- 0.24705882352941178,
278
- 0.5294117647058824,
279
- 0.5019607843137255,
280
- 0.8313725490196079,
281
- 0.7058823529411765,
282
- 0.6980392156862745,
283
- 0.9764705882352941,
284
- 0.9450980392156862,
285
- 0.43137254901960786,
286
- 0.00784313725490196,
287
- 0.33725490196078434,
288
- 0.9333333333333333,
289
- 0.1568627450980392,
290
- 0.792156862745098,
291
- 0.21568627450980393,
292
- 0.0784313725490196,
293
- 0.0196078431372549,
294
- 0.796078431372549,
295
- 0.10980392156862745,
296
- 0.4627450980392157,
297
- 0.9490196078431372,
298
- 0.24313725490196078,
299
- 0.8117647058823529,
300
- 0.6,
301
- 0.06274509803921569,
302
- 0.1411764705882353,
303
- 0.1450980392156863,
304
- 0.3176470588235294,
305
- 0.1568627450980392,
306
- 0.8666666666666667,
307
- 0.3568627450980392,
308
- 0.2823529411764706,
309
- 0.24705882352941178,
310
- 0.5294117647058824,
311
- 0.5019607843137255,
312
- 0.8313725490196079,
313
- 0.7058823529411765,
314
- 0.6980392156862745,
315
- 0.9764705882352941,
316
- 0.9450980392156862,
317
- 0.43137254901960786,
318
- 0.00784313725490196,
319
- 0.33725490196078434,
320
- 0.9333333333333333,
321
- 0.1568627450980392,
322
- 0.792156862745098,
323
- 0.21568627450980393,
324
- 0.0784313725490196,
325
- 0.0196078431372549,
326
- 0.796078431372549,
327
- 0.10980392156862745,
328
- 0.4627450980392157,
329
- 0.9490196078431372,
330
- 0.24313725490196078,
331
- 0.8117647058823529,
332
- 0.6,
333
- 0.06274509803921569,
334
- 0.1411764705882353,
335
- 0.1450980392156863,
336
- 0.3176470588235294,
337
- 0.1568627450980392,
338
- 0.8666666666666667,
339
- 0.3568627450980392,
340
- 0.2823529411764706,
341
- 0.24705882352941178,
342
- 0.5294117647058824,
343
- 0.5019607843137255,
344
- 0.8313725490196079,
345
- 0.7058823529411765,
346
- 0.6980392156862745,
347
- 0.9764705882352941,
348
- 0.9450980392156862,
349
- 0.43137254901960786,
350
- 0.00784313725490196,
351
- 0.33725490196078434,
352
- 0.9333333333333333,
353
- 0.1568627450980392,
354
- 0.792156862745098,
355
- 0.21568627450980393,
356
- 0.0784313725490196,
357
- 0.0196078431372549,
358
- 0.796078431372549,
359
- 0.10980392156862745,
360
- 0.4627450980392157,
361
- 0.9490196078431372,
362
- 0.24313725490196078,
363
- 0.8117647058823529,
364
- 0.6,
365
- 0.06274509803921569,
366
- 0.1411764705882353,
367
- 0.1450980392156863,
368
- 0.3176470588235294,
369
- 0.1568627450980392,
370
- 0.8666666666666667,
371
- 0.3568627450980392,
372
- 0.2823529411764706,
373
- 0.24705882352941178,
374
- 0.5294117647058824,
375
- 0.5019607843137255,
376
- 0.8313725490196079,
377
- 0.7058823529411765,
378
- 0.6980392156862745,
379
- 0.9764705882352941,
380
- 0.9450980392156862,
381
- 0.43137254901960786,
382
- 0.00784313725490196,
383
- 0.33725490196078434,
384
- 0.9333333333333333,
385
- 0.1568627450980392,
386
- 0.792156862745098,
387
- 0.21568627450980393,
388
- 0.0784313725490196,
389
- 0.0196078431372549,
390
- 0.796078431372549,
391
- 0.10980392156862745,
392
- 0.4627450980392157,
393
- 0.9490196078431372,
394
- 0.24313725490196078,
395
- 0.8117647058823529,
396
- 0.6,
397
- 0.06274509803921569,
398
- 0.1411764705882353,
399
- 0.1450980392156863,
400
- 0.3176470588235294,
401
- 0.1568627450980392,
402
- 0.8666666666666667,
403
- 0.3568627450980392,
404
- 0.2823529411764706,
405
- 0.24705882352941178,
406
- 0.5294117647058824,
407
- 0.5019607843137255,
408
- 0.8313725490196079,
409
- 0.7058823529411765,
410
- 0.6980392156862745,
411
- 0.9764705882352941,
412
- 0.9450980392156862,
413
- 0.43137254901960786,
414
- 0.00784313725490196,
415
- 0.33725490196078434,
416
- 0.9333333333333333,
417
- 0.1568627450980392,
418
- 0.792156862745098,
419
- 0.21568627450980393,
420
- 0.0784313725490196,
421
- 0.0196078431372549,
422
- 0.796078431372549,
423
- 0.10980392156862745,
424
- 0.4627450980392157,
425
- 0.9490196078431372,
426
- 0.24313725490196078,
427
- 0.8117647058823529,
428
- 0.6,
429
- 0.06274509803921569,
430
- 0.1411764705882353,
431
- 0.1450980392156863,
432
- 0.3176470588235294,
433
- 0.1568627450980392,
434
- 0.8666666666666667,
435
- 0.3568627450980392,
436
- 0.2823529411764706,
437
- 0.24705882352941178,
438
- 0.5294117647058824,
439
- 0.5019607843137255,
440
- 0.8313725490196079,
441
- 0.7058823529411765,
442
- 0.6980392156862745,
443
- 0.9764705882352941,
444
- 0.9450980392156862,
445
- 0.43137254901960786,
446
- 0.00784313725490196,
447
- 0.33725490196078434,
448
- 0.9333333333333333,
449
- 0.1568627450980392,
450
- 0.792156862745098,
451
- 0.21568627450980393,
452
- 0.0784313725490196,
453
- 0.0196078431372549,
454
- 0.796078431372549,
455
- 0.10980392156862745,
456
- 0.4627450980392157,
457
- 0.9490196078431372,
458
- 0.24313725490196078,
459
- 0.8117647058823529,
460
- 0.6,
461
- 0.06274509803921569,
462
- 0.1411764705882353,
463
- 0.1450980392156863,
464
- 0.3176470588235294,
465
- 0.1568627450980392,
466
- 0.8666666666666667,
467
- 0.3568627450980392,
468
- 0.2823529411764706,
469
- 0.24705882352941178,
470
- 0.5294117647058824,
471
- 0.5019607843137255,
472
- 0.8313725490196079,
473
- 0.7058823529411765,
474
- 0.6980392156862745,
475
- 0.9764705882352941,
476
- 0.9450980392156862,
477
- 0.43137254901960786,
478
- 0.00784313725490196,
479
- 0.33725490196078434,
480
- 0.9333333333333333,
481
- 0.1568627450980392,
482
- 0.792156862745098,
483
- 0.21568627450980393,
484
- 0.0784313725490196,
485
- 0.0196078431372549,
486
- 0.796078431372549,
487
- 0.10980392156862745,
488
- 0.4627450980392157,
489
- 0.9490196078431372,
490
- 0.24313725490196078,
491
- 0.8117647058823529,
492
- 0.6,
493
- 0.06274509803921569,
494
- 0.1411764705882353,
495
- 0.1450980392156863,
496
- 0.3176470588235294,
497
- 0.1568627450980392,
498
- 0.8666666666666667,
499
- 0.3568627450980392,
500
- 0.2823529411764706,
501
- 0.24705882352941178,
502
- 0.5294117647058824,
503
- 0.5019607843137255,
504
- 0.8313725490196079,
505
- 0.7058823529411765,
506
- 0.6980392156862745,
507
- 0.9764705882352941,
508
- 0.9450980392156862,
509
- 0.43137254901960786,
510
- 0.00784313725490196,
511
- 0.33725490196078434,
512
- 0.9333333333333333,
513
- 0.1568627450980392,
514
- 0.792156862745098
515
- ]
2
+ "sample1.txt": true
516
3
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@o-lang/semantic-doc-search",
3
- "version": "1.0.23",
3
+ "version": "1.0.24",
4
4
  "description": "O-lang Semantic Document Search Resolver with hybrid search, embeddings, rerank, and streaming.",
5
5
  "main": "src/index.js",
6
6
  "type": "commonjs",
@@ -13,6 +13,7 @@
13
13
  },
14
14
  "dependencies": {
15
15
  "@anthropic-ai/sdk": "*",
16
+ "@xenova/transformers": "^2.17.2",
16
17
  "axios": "^1.7.2",
17
18
  "docx": "^7.0.0",
18
19
  "dotenv": "^17.2.3",