@delali/narsil 0.4.1 → 0.4.2-next.273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +9 -3
- package/dist/adapters/gcs.d.ts +1 -1
- package/dist/adapters/indexeddb.d.ts +6 -1
- package/dist/adapters/indexeddb.mjs +20 -26
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/adapters/s3.d.ts +1 -1
- package/dist/{adapters-pljAqmrC.d.ts → adapters-8LcdNB77.d.ts} +18 -1
- package/dist/browser/{chunk-QAOVAYAG.mjs → chunk-56UNHLAF.mjs} +1 -1
- package/dist/browser/{chunk-GBJSEHVJ.mjs → chunk-EKISQBV2.mjs} +283 -244
- package/dist/browser/{chunk-YINZFPLT.mjs → chunk-VAPCFSMX.mjs} +1 -1
- package/dist/browser/{chunk-4E5APHTM.mjs → chunk-VIVG3OXM.mjs} +1 -1
- package/dist/browser/{durability-config-75NSFCQN.mjs → durability-config-UP3OTUMB.mjs} +34 -2
- package/dist/browser/{msgpack-leaf-44XNWSSY.mjs → msgpack-leaf-76Q5SZ5S.mjs} +23 -10
- package/dist/browser/{optional-reads-6LN3HMUH.mjs → optional-reads-NQO5DKM4.mjs} +18 -16
- package/dist/browser/{optional-writes-76DWMRQH.mjs → optional-writes-3HKIVZFU.mjs} +295 -106
- package/dist/browser/{segment-codec-leaf-A427QJUX.mjs → segment-codec-leaf-GXSRX5VT.mjs} +47 -30
- package/dist/browser/{storage-leaf-5UTR5K7J.mjs → storage-leaf-42IZEOJJ.mjs} +1473 -305
- package/dist/{capabilities-BOFFMx84.d.ts → capabilities-BoWiLNv7.d.ts} +76 -61
- package/dist/{chunk-Z7XSHKOF.mjs → chunk-35RWLUVY.mjs} +63 -144
- package/dist/{chunk-6GHX35ZM.mjs → chunk-43T3P3BE.mjs} +1 -1
- package/dist/{chunk-IL5DRCNU.mjs → chunk-4JLHHEBN.mjs} +1 -1
- package/dist/{chunk-DKIMHIBC.mjs → chunk-4YIVY3PB.mjs} +2 -2
- package/dist/{react-native/chunks/chunk-ZUJCUPVS.mjs → chunk-5SHWA6PE.mjs} +1 -1
- package/dist/{chunk-2K24MEYV.mjs → chunk-6LVQ3GLT.mjs} +3 -3
- package/dist/{chunk-DDUXEJXW.mjs → chunk-6SIEVWMG.mjs} +45 -22
- package/dist/{chunk-WM5KJSQV.mjs → chunk-ACNHJOGC.mjs} +1 -1
- package/dist/{chunk-755NM4PL.mjs → chunk-BWPVDIRN.mjs} +183 -21
- package/dist/{chunk-SMQPZHPS.mjs → chunk-C57GWAXN.mjs} +1 -1
- package/dist/{chunk-357KIU5E.mjs → chunk-DRZP5MSG.mjs} +5 -5
- package/dist/{chunk-MYN4K5IS.mjs → chunk-DZMD4OMD.mjs} +1 -1
- package/dist/{chunk-7P5YWDRQ.mjs → chunk-EY5U4P5V.mjs} +1 -1
- package/dist/{chunk-X3EQCQLP.mjs → chunk-EZJ6HBG7.mjs} +2 -2
- package/dist/{chunk-WC5KAWXQ.mjs → chunk-FE2XILWN.mjs} +4 -4
- package/dist/{chunk-CERPUGRK.mjs → chunk-G3KVEHIW.mjs} +2 -2
- package/dist/{chunk-B3MO37YF.mjs → chunk-GTGQLCEI.mjs} +1 -1
- package/dist/{chunk-MSJD74GN.mjs → chunk-H23FB267.mjs} +5 -5
- package/dist/{chunk-UNEEFTEU.mjs → chunk-HZ3OAUTL.mjs} +2 -1
- package/dist/{chunk-M3B7SGJB.mjs → chunk-KL4ORYJW.mjs} +6 -6
- package/dist/{chunk-WTMCGZ74.mjs → chunk-KXA77DPZ.mjs} +2 -2
- package/dist/{chunk-6I6OO6B2.mjs → chunk-LRIJ7U3T.mjs} +4 -4
- package/dist/{chunk-3IE3A74H.mjs → chunk-PNFP5THV.mjs} +3 -3
- package/dist/{react-native/chunks/chunk-4FUCRLSA.mjs → chunk-PQWK7KXS.mjs} +1 -1
- package/dist/{chunk-P4F4UWLC.mjs → chunk-PSO7V7VW.mjs} +17 -3
- package/dist/{chunk-OGYSPNL2.mjs → chunk-QNZJYMCF.mjs} +7 -7
- package/dist/{chunk-DBC4EJWB.mjs → chunk-R3PFPFME.mjs} +1 -1
- package/dist/{chunk-MSIN4NM3.mjs → chunk-S3JHYPH7.mjs} +6 -1
- package/dist/{chunk-MI6UZGUR.mjs → chunk-SPA7GXST.mjs} +3 -3
- package/dist/{chunk-PP3W5PQQ.mjs → chunk-WYH3QHBK.mjs} +698 -451
- package/dist/{chunk-D456ZFPQ.mjs → chunk-X5V25BPN.mjs} +2 -2
- package/dist/{chunk-GXCDWLQI.mjs → chunk-Y7GXGJFZ.mjs} +23 -10
- package/dist/{chunk-NCIGNQJE.mjs → chunk-YGRJLI2Z.mjs} +930 -387
- package/dist/{react-native/chunks/chunk-MMPBO4WR.mjs → chunk-ZO7VLAIT.mjs} +2 -2
- package/dist/{chunk-AXDB6KUQ.mjs → chunk-ZTB7AWTW.mjs} +4 -4
- package/dist/chunk-ZYEEOZTT.mjs +52 -0
- package/dist/{client-BhNxHtcZ.d.ts → client-B0NPZSKY.d.ts} +4 -4
- package/dist/client.d.ts +9 -9
- package/dist/client.mjs +2 -2
- package/dist/{config-P9zW3Zz7.d.ts → config-B3-sw887.d.ts} +3 -3
- package/dist/{config-BJSL7piq.d.ts → config-DL_nkguq.d.ts} +4 -4
- package/dist/{direct-executor-GWJXDFLD.mjs → direct-executor-7G57AYXH.mjs} +4 -4
- package/dist/distribution/coordinator/etcd.d.ts +3 -3
- package/dist/distribution/coordinator/etcd.mjs +33 -3
- package/dist/distribution/coordinator/in-memory.d.ts +3 -3
- package/dist/distribution/transport/grpc.mjs +4 -4
- package/dist/distribution/transport/in-memory.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +12 -7
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +167 -91
- package/dist/{durability-config-VWS2K24E.mjs → durability-config-HWKWP35M.mjs} +34 -2
- package/dist/embeddings/openai.d.ts +7 -3
- package/dist/embeddings/openai.mjs +37 -31
- package/dist/{engine-DoFnkRoF.d.ts → engine-DGG_yX0p.d.ts} +3 -3
- package/dist/{index-B_vmxpgW.d.ts → index-fX_FbuRT.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +4 -4
- package/dist/index.react-native.mjs +4 -4
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/{long-query-thread-GNTDYDOZ.mjs → long-query-thread-KPN7NSOC.mjs} +14 -14
- package/dist/msgpack-leaf-LIT7TKZW.mjs +9 -0
- package/dist/{optional-reads-UCS23AWP.mjs → optional-reads-ESDVS225.mjs} +18 -16
- package/dist/{optional-writes-2AHJCGTY.mjs → optional-writes-CSULLISE.mjs} +9 -8
- package/dist/{paged-partition-archive-GBAHDT55.mjs → paged-partition-archive-BKQADXOT.mjs} +13 -13
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +2 -2
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/{query-actions-IDCUA7UV.mjs → query-actions-HTSJKZRU.mjs} +4 -4
- package/dist/query-suggestions/merge-worker.mjs +7 -7
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +3 -3
- package/dist/query-suggestions.mjs +5 -5
- package/dist/react-native/chunks/{chunk-MRH36RN3.mjs → chunk-AOLLV37A.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-UO44MA5M.mjs → chunk-DN4FT62B.mjs} +23 -10
- package/dist/react-native/chunks/{chunk-TG56N5JC.mjs → chunk-EF4Y5C3Y.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-HTBY4GON.mjs → chunk-ESVBC3AT.mjs} +1 -1
- package/dist/{chunk-F26DOOUH.mjs → react-native/chunks/chunk-H6LK5WCF.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-LPX26NNL.mjs → chunk-JEYZQA4Z.mjs} +699 -451
- package/dist/react-native/chunks/{chunk-AJAH6X7W.mjs → chunk-KJQ3BUVM.mjs} +4 -4
- package/dist/{chunk-UHLFCGZ3.mjs → react-native/chunks/chunk-QYVRD6C5.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-SCGBU47K.mjs → chunk-X3VY2YT4.mjs} +2 -2
- package/dist/{chunk-R2PJKSNL.mjs → react-native/chunks/chunk-ZF3VSGEI.mjs} +1 -1
- package/dist/react-native/chunks/{durability-config-FCSD4MXY.mjs → durability-config-HJETGZVB.mjs} +34 -2
- package/dist/react-native/chunks/msgpack-leaf-FOUPTT7J.mjs +7 -0
- package/dist/react-native/chunks/{optional-reads-4XCJ34QG.mjs → optional-reads-TN3PT6M4.mjs} +18 -16
- package/dist/react-native/chunks/{optional-writes-XYZVTBFB.mjs → optional-writes-MGFFGKYQ.mjs} +8 -7
- package/dist/react-native/chunks/{restore-queue-NOKQZJ2Z.mjs → restore-queue-MVJTP3E4.mjs} +7 -7
- package/dist/react-native/chunks/{segment-codec-leaf-3ZXBJVKH.mjs → segment-codec-leaf-PSWHL55A.mjs} +5 -5
- package/dist/react-native/chunks/{segment-merge-leaf-3Z7MMPWE.mjs → segment-merge-leaf-WOJWSDLW.mjs} +6 -6
- package/dist/react-native/chunks/{storage-leaf-KJOX3MMW.mjs → storage-leaf-ZUN6Y44X.mjs} +1481 -314
- package/dist/react-native/chunks/{store-leaf-QGLVSO5X.mjs → store-leaf-K5XG46FV.mjs} +6 -6
- package/dist/react-native/chunks/{suggestions-leaf-6GI5ZSW6.mjs → suggestions-leaf-KDPM3P27.mjs} +6 -6
- package/dist/react-native/chunks/{worker-copies-ALOD7TSP.mjs → worker-copies-FVFMBBH6.mjs} +5 -5
- package/dist/react-native/partial-words.mjs +4 -4
- package/dist/react-native/pattern.mjs +2 -2
- package/dist/react-native/query-suggestions.mjs +4 -4
- package/dist/react-native/sync.mjs +4 -4
- package/dist/react-native/synonyms.mjs +4 -4
- package/dist/react.d.ts +10 -10
- package/dist/react.mjs +0 -2
- package/dist/{rerank-HB6DU6GS.mjs → rerank-3QEJNMFH.mjs} +4 -4
- package/dist/rerankers/http.d.ts +1 -1
- package/dist/rerankers/http.mjs +19 -15
- package/dist/{restore-queue-VUQHGCXE.mjs → restore-queue-Q76Q4S35.mjs} +7 -7
- package/dist/{schema-CzG9OUw5.d.ts → schema-CFnQ-WI3.d.ts} +1 -1
- package/dist/{segment-codec-leaf-TW2FPAAT.mjs → segment-codec-leaf-BSDDR7DQ.mjs} +5 -5
- package/dist/{segment-merge-leaf-BS3QV5PZ.mjs → segment-merge-leaf-6TT66HC5.mjs} +5 -5
- package/dist/server.d.ts +86 -12
- package/dist/server.mjs +2243 -1918
- package/dist/{storage-leaf-5IG2HPNY.mjs → storage-leaf-R5JQ2X2M.mjs} +1193 -219
- package/dist/{store-leaf-ZWSOK2DV.mjs → store-leaf-NCGETZYY.mjs} +6 -6
- package/dist/{suggestions-leaf-MUZNKQVF.mjs → suggestions-leaf-TX3GTD6S.mjs} +8 -8
- package/dist/sync/filesystem.d.ts +4 -4
- package/dist/sync/filesystem.mjs +12 -12
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +4 -4
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +4 -4
- package/dist/sync.d.ts +5 -5
- package/dist/sync.mjs +4 -4
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +4 -4
- package/dist/{tasks-DrLiolTG.d.ts → tasks-B2MDjPL0.d.ts} +4 -4
- package/dist/{thread-6ZBCENAP.mjs → thread-VLVPAUWC.mjs} +13 -13
- package/dist/{types-pS-mAJmX.d.ts → types-DGjk684l.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +2 -2
- package/dist/{worker-copies-32HWFUH7.mjs → worker-copies-NZSBQZKG.mjs} +7 -7
- package/dist/workers/entry.mjs +16 -16
- package/package.json +12 -12
- package/dist/chunk-HLPRHOSO.mjs +0 -17
- package/dist/msgpack-leaf-H5ZYW7S7.mjs +0 -9
- package/dist/react-native/chunks/msgpack-leaf-JMNGSTRW.mjs +0 -7
|
@@ -55,11 +55,37 @@ function filesystemBackedDirectory(config) {
|
|
|
55
55
|
}
|
|
56
56
|
return null;
|
|
57
57
|
}
|
|
58
|
+
function indexedDBLocationOf(config) {
|
|
59
|
+
const location = config.persistence?.indexedDB;
|
|
60
|
+
if (location === void 0) return null;
|
|
61
|
+
const { databaseName, storeName } = location;
|
|
62
|
+
if (typeof databaseName !== "string" || databaseName.length === 0) {
|
|
63
|
+
throw configError("persistence.indexedDB.databaseName must be a non-empty string");
|
|
64
|
+
}
|
|
65
|
+
if (typeof storeName !== "string" || storeName.length === 0) {
|
|
66
|
+
throw configError("persistence.indexedDB.storeName must be a non-empty string");
|
|
67
|
+
}
|
|
68
|
+
return { databaseName, storeName };
|
|
69
|
+
}
|
|
70
|
+
function requireNoLogFieldsOnSharedStore(durability) {
|
|
71
|
+
for (const field of WAL_ONLY_FIELDS) {
|
|
72
|
+
if (durability?.[field] !== void 0) {
|
|
73
|
+
throw configError(
|
|
74
|
+
`durability.${field} applies to the write-ahead log, which an IndexedDB store that an invalidation adapter shares between tabs cannot keep. Remove durability.${field}, or remove the invalidation adapter`
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
58
79
|
function resolveDurabilityTier(config) {
|
|
59
80
|
const filesystemDirectory = filesystemBackedDirectory(config);
|
|
81
|
+
const indexedDB = filesystemDirectory === null ? indexedDBLocationOf(config) : null;
|
|
60
82
|
if (config.durability) {
|
|
61
83
|
requireValidDurabilityConfig(config.durability);
|
|
62
84
|
}
|
|
85
|
+
if (indexedDB !== null && config.persistence !== void 0 && config.invalidation !== void 0 && config.durability?.tier === void 0) {
|
|
86
|
+
requireNoLogFieldsOnSharedStore(config.durability);
|
|
87
|
+
return { kind: "snapshot", config: { ...config.durability }, adapter: config.persistence };
|
|
88
|
+
}
|
|
63
89
|
if (config.durability?.tier === "snapshot") {
|
|
64
90
|
if (config.persistence === void 0) {
|
|
65
91
|
throw configError(
|
|
@@ -75,13 +101,16 @@ function resolveDurabilityTier(config) {
|
|
|
75
101
|
if (filesystemDirectory !== null) {
|
|
76
102
|
return { kind: "wal", config: { ...config.durability, directory: filesystemDirectory } };
|
|
77
103
|
}
|
|
104
|
+
if (indexedDB !== null) {
|
|
105
|
+
return { kind: "wal", config: { ...config.durability }, indexedDB };
|
|
106
|
+
}
|
|
78
107
|
if (config.persistence !== void 0) {
|
|
79
108
|
throw configError(
|
|
80
|
-
|
|
109
|
+
'WAL durability requires a filesystem directory or the IndexedDB persistence adapter, and the configured persistence adapter is neither. Use one of those adapters, set durability.directory, or set durability.tier to "snapshot"'
|
|
81
110
|
);
|
|
82
111
|
}
|
|
83
112
|
throw configError(
|
|
84
|
-
"Durability requires a directory. Set durability.directory explicitly, or configure a filesystem persistence adapter"
|
|
113
|
+
"Durability requires a directory. Set durability.directory explicitly, or configure a filesystem or IndexedDB persistence adapter"
|
|
85
114
|
);
|
|
86
115
|
}
|
|
87
116
|
if (config.persistence === void 0) {
|
|
@@ -90,6 +119,9 @@ function resolveDurabilityTier(config) {
|
|
|
90
119
|
if (filesystemDirectory !== null) {
|
|
91
120
|
return { kind: "wal", config: { directory: filesystemDirectory } };
|
|
92
121
|
}
|
|
122
|
+
if (indexedDB !== null) {
|
|
123
|
+
return { kind: "wal", config: {}, indexedDB };
|
|
124
|
+
}
|
|
93
125
|
return { kind: "snapshot", config: {}, adapter: config.persistence };
|
|
94
126
|
}
|
|
95
127
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter } from '../adapters-
|
|
1
|
+
import { E as EmbeddingAdapter } from '../adapters-8LcdNB77.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* How {@link createOpenAIEmbedding} reaches an embeddings endpoint.
|
|
@@ -8,8 +8,12 @@ import { E as EmbeddingAdapter } from '../adapters-pljAqmrC.js';
|
|
|
8
8
|
interface OpenAIEmbeddingConfig {
|
|
9
9
|
/** This points the adapter at OpenAI, at Azure, or at a compatible service you host. */
|
|
10
10
|
baseUrl: string;
|
|
11
|
-
/**
|
|
12
|
-
|
|
11
|
+
/**
|
|
12
|
+
* This holds the key, or a function returning it, which is how you rotate a key or fetch a short-lived token per
|
|
13
|
+
* request. Leave it out for a service that you host without a key, and the adapter then sends no `Authorization`
|
|
14
|
+
* header.
|
|
15
|
+
*/
|
|
16
|
+
apiKey?: string | (() => string | Promise<string>);
|
|
13
17
|
/** The adapter embeds with this model. */
|
|
14
18
|
model: string;
|
|
15
19
|
/** The model returns vectors this long, which must match the schema's `vector[N]`. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { buildSignal, postJsonWithRetry, resolveApiKey } from '../chunk-PSO7V7VW.mjs';
|
|
2
2
|
import { NarsilError, ErrorCodes } from '../chunk-ZKOQR7EA.mjs';
|
|
3
3
|
|
|
4
4
|
// src/embeddings/constants.ts
|
|
@@ -6,28 +6,35 @@ var DEFAULT_REQUEST_TIMEOUT_MS = 3e4;
|
|
|
6
6
|
var MAX_INPUTS_PER_REQUEST = 2048;
|
|
7
7
|
|
|
8
8
|
// src/embeddings/openai.ts
|
|
9
|
-
function parseEmbeddingItems(data) {
|
|
9
|
+
function parseEmbeddingItems(data, count, dimensions) {
|
|
10
10
|
if (!Array.isArray(data)) {
|
|
11
11
|
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, 'OpenAI response missing "data" array');
|
|
12
12
|
}
|
|
13
|
+
if (data.length !== count) {
|
|
14
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `OpenAI returned ${data.length} embeddings for ${count} inputs`);
|
|
15
|
+
}
|
|
13
16
|
const items = [];
|
|
17
|
+
const indexes = /* @__PURE__ */ new Set();
|
|
14
18
|
for (let i = 0; i < data.length; i++) {
|
|
15
19
|
const item = data[i];
|
|
16
20
|
if (item === void 0 || item === null || typeof item !== "object") {
|
|
17
21
|
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, `OpenAI response item at position ${i} is not an object`);
|
|
18
22
|
}
|
|
19
|
-
if (typeof item.index !== "number") {
|
|
23
|
+
if (typeof item.index !== "number" || !Number.isSafeInteger(item.index) || item.index < 0 || item.index >= count || indexes.has(item.index)) {
|
|
20
24
|
throw new NarsilError(
|
|
21
25
|
ErrorCodes.EMBEDDING_FAILED,
|
|
22
|
-
`OpenAI response item at position ${i}
|
|
26
|
+
`OpenAI response item at position ${i} has an invalid or duplicate index`
|
|
23
27
|
);
|
|
24
28
|
}
|
|
25
|
-
if (!Array.isArray(item.embedding)
|
|
29
|
+
if (!Array.isArray(item.embedding) || item.embedding.length !== dimensions || item.embedding.some(
|
|
30
|
+
(value) => typeof value !== "number" || !Number.isFinite(value) || !Number.isFinite(Math.fround(value))
|
|
31
|
+
)) {
|
|
26
32
|
throw new NarsilError(
|
|
27
33
|
ErrorCodes.EMBEDDING_FAILED,
|
|
28
|
-
`OpenAI response item at position ${i}
|
|
34
|
+
`OpenAI response item at position ${i} has an invalid embedding`
|
|
29
35
|
);
|
|
30
36
|
}
|
|
37
|
+
indexes.add(item.index);
|
|
31
38
|
items.push({ index: item.index, embedding: item.embedding });
|
|
32
39
|
}
|
|
33
40
|
return items;
|
|
@@ -60,7 +67,7 @@ function createOpenAIEmbedding(config) {
|
|
|
60
67
|
if (typeof config.apiKey === "string" && config.apiKey.trim().length === 0) {
|
|
61
68
|
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "OpenAI embedding adapter requires a non-empty apiKey");
|
|
62
69
|
}
|
|
63
|
-
if (typeof config.apiKey !== "string" && typeof config.apiKey !== "function") {
|
|
70
|
+
if (config.apiKey !== void 0 && typeof config.apiKey !== "string" && typeof config.apiKey !== "function") {
|
|
64
71
|
throw new NarsilError(
|
|
65
72
|
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
66
73
|
"OpenAI embedding adapter apiKey must be a string or a function returning a string"
|
|
@@ -71,12 +78,14 @@ function createOpenAIEmbedding(config) {
|
|
|
71
78
|
const dimensionCount = config.dimensions;
|
|
72
79
|
const timeoutMs = config.timeout ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
73
80
|
const maxRetries = config.maxRetries ?? 3;
|
|
81
|
+
const apiKeySource = config.apiKey;
|
|
82
|
+
const currentApiKey = () => apiKeySource === void 0 ? Promise.resolve(null) : resolveApiKey(apiKeySource, "OpenAI embedding adapter", ErrorCodes.EMBEDDING_CONFIG_INVALID);
|
|
74
83
|
return {
|
|
75
84
|
get dimensions() {
|
|
76
85
|
return dimensionCount;
|
|
77
86
|
},
|
|
78
87
|
async embed(input, _purpose, signal) {
|
|
79
|
-
const apiKey = await
|
|
88
|
+
const apiKey = await currentApiKey();
|
|
80
89
|
const body = JSON.stringify({ input: [input], model, dimensions: dimensionCount });
|
|
81
90
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
82
91
|
const result = await postJsonWithRetry({
|
|
@@ -84,11 +93,11 @@ function createOpenAIEmbedding(config) {
|
|
|
84
93
|
body,
|
|
85
94
|
apiKey,
|
|
86
95
|
maxRetries,
|
|
87
|
-
signal: combinedSignal,
|
|
96
|
+
signal: combinedSignal.signal,
|
|
88
97
|
service: "OpenAI",
|
|
89
98
|
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
90
|
-
});
|
|
91
|
-
const items = parseEmbeddingItems(result.data);
|
|
99
|
+
}).finally(combinedSignal.dispose);
|
|
100
|
+
const items = parseEmbeddingItems(result.data, 1, dimensionCount);
|
|
92
101
|
if (items.length === 0) {
|
|
93
102
|
throw new NarsilError(
|
|
94
103
|
ErrorCodes.EMBEDDING_FAILED,
|
|
@@ -99,7 +108,7 @@ function createOpenAIEmbedding(config) {
|
|
|
99
108
|
},
|
|
100
109
|
async embedBatch(inputs, _purpose, signal) {
|
|
101
110
|
if (inputs.length === 0) return [];
|
|
102
|
-
const apiKey = await
|
|
111
|
+
const apiKey = await currentApiKey();
|
|
103
112
|
const combinedSignal = buildSignal(signal, timeoutMs);
|
|
104
113
|
const executeChunk = async (chunk) => {
|
|
105
114
|
const body = JSON.stringify({ input: chunk, model, dimensions: dimensionCount });
|
|
@@ -108,33 +117,30 @@ function createOpenAIEmbedding(config) {
|
|
|
108
117
|
body,
|
|
109
118
|
apiKey,
|
|
110
119
|
maxRetries,
|
|
111
|
-
signal: combinedSignal,
|
|
120
|
+
signal: combinedSignal.signal,
|
|
112
121
|
service: "OpenAI",
|
|
113
122
|
failureCode: ErrorCodes.EMBEDDING_FAILED
|
|
114
123
|
});
|
|
115
|
-
const items = parseEmbeddingItems(result.data);
|
|
116
|
-
if (items.length !== chunk.length) {
|
|
117
|
-
throw new NarsilError(
|
|
118
|
-
ErrorCodes.EMBEDDING_FAILED,
|
|
119
|
-
`OpenAI returned ${items.length} embeddings for ${chunk.length} inputs`,
|
|
120
|
-
{ expected: chunk.length, actual: items.length }
|
|
121
|
-
);
|
|
122
|
-
}
|
|
124
|
+
const items = parseEmbeddingItems(result.data, chunk.length, dimensionCount);
|
|
123
125
|
const sorted = items.slice().sort((a, b) => a.index - b.index);
|
|
124
126
|
return sorted.map((item) => new Float32Array(item.embedding));
|
|
125
127
|
};
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
128
|
+
try {
|
|
129
|
+
if (inputs.length <= MAX_INPUTS_PER_REQUEST) {
|
|
130
|
+
return await executeChunk(inputs);
|
|
131
|
+
}
|
|
132
|
+
const results = [];
|
|
133
|
+
for (let start = 0; start < inputs.length; start += MAX_INPUTS_PER_REQUEST) {
|
|
134
|
+
const chunk = inputs.slice(start, start + MAX_INPUTS_PER_REQUEST);
|
|
135
|
+
const chunkResults = await executeChunk(chunk);
|
|
136
|
+
for (const vec of chunkResults) {
|
|
137
|
+
results.push(vec);
|
|
138
|
+
}
|
|
135
139
|
}
|
|
140
|
+
return results;
|
|
141
|
+
} finally {
|
|
142
|
+
combinedSignal.dispose();
|
|
136
143
|
}
|
|
137
|
-
return results;
|
|
138
144
|
},
|
|
139
145
|
async shutdown() {
|
|
140
146
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-
|
|
2
|
-
import { A as AnyDocument, B as BM25Params, d as ScoringMode, S as SchemaDefinition, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-
|
|
1
|
+
import { E as EmbeddingAdapter, R as RerankAdapter } from './adapters-8LcdNB77.js';
|
|
2
|
+
import { A as AnyDocument, B as BM25Params, d as ScoringMode, S as SchemaDefinition, a as InsertOptions, W as WriteOptions, I as IndexConfig, P as PartitionConfig } from './schema-CFnQ-WI3.js';
|
|
3
3
|
import { h as QuerySuggestionsBuildEvent, j as QuerySuggestionsWarningEvent } from './query-suggestions-CBYkrQ5T.js';
|
|
4
4
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
5
5
|
|
|
@@ -2830,4 +2830,4 @@ interface Narsil extends IndexLifecycleOperations, DocumentWriteOperations, Syno
|
|
|
2830
2830
|
shutdown(): Promise<void>;
|
|
2831
2831
|
}
|
|
2832
2832
|
|
|
2833
|
-
export { type FacetResult as $, type
|
|
2833
|
+
export { type FacetResult as $, type AnalysisChange as A, type BatchResult as B, type Change as C, type NarsilEventMap as D, type ErrorCode as E, type FollowChangesOptions as F, type GroupResult as G, type Hit as H, type IndexInfo as I, type GroupConfig as J, type AdapterRegistrationOperations as K, type ListParams as L, type MemoryStats as M, NarsilError as N, type AnalysisOperations as O, type PartitionStatsResult as P, type QueryParams as Q, type ArrayFilter as R, type ServerErrorCode as S, type BM25Operations as T, type ByteSource as U, type VectorMaintenanceResult as V, type ComparisonFilter as W, type DocumentImportResult as X, type DocumentProjection as Y, type DocumentWriteOperations as Z, type FacetConfig as _, type ChangePosition as a, type FieldFilter as a0, type FilterExpression as a1, type GeoFilter as a2, type GeoPolygonFilter as a3, type GeoRadiusFilter as a4, type GroupReducer as a5, type HeldCheckpointReport as a6, type HighlightConfig as a7, type HighlightMatch as a8, type HybridConfig as a9, type IndexLifecycleOperations as aa, type ListedDocument as ab, type PresenceFilter as ac, type ProcessMemoryReport as ad, type QueryCoverage as ae, type RerankConfig as af, type ScoreComponents as ag, type SearchMode as ah, type SortField as ai, type SortSpec as aj, type StringFilter as ak, type SyncOperations as al, type SynonymOperations as am, type TermMatchPolicy as an, type VectorMaintenanceOperations as ao, type VectorQueryConfig as ap, type WorkerCopyReport as aq, type ChangeType as b, type ChangesBatch as c, type ChangesEvent as d, type ChangesOperations as e, type ChangesResync as f, type ChangesSetting as g, type ChangesStatus as h, type ChangesSubscribed as i, type ClientErrorCode as j, ClientErrorCodes as k, ErrorCodes as l, type NarsilErrorCode as m, ServerErrorCodes as n, type SnapshotOptions as o, type SyncSetting as p, type SyncStatus as q, isNarsilError as r, type Narsil as s, type QueryResult as t, type NarsilEventHandlers as u, type ListResult as v, type SuggestParams as w, type SuggestResult as x, type PreflightResult as y, type IndexStats as z };
|
package/dist/index.browser.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-
|
|
2
|
-
import './browser/chunk-
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './browser/chunk-EKISQBV2.mjs';
|
|
2
|
+
import './browser/chunk-56UNHLAF.mjs';
|
|
3
3
|
import './browser/chunk-YJHGOJAC.mjs';
|
|
4
|
-
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './browser/chunk-
|
|
4
|
+
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './browser/chunk-VAPCFSMX.mjs';
|
package/dist/index.d.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import { e as StopWordOverride, C as CustomTokenizer } from './schema-
|
|
2
|
-
export { A as AnyDocument, c as BM25FieldParams, B as BM25Params, E as EmbeddingFieldConfig, f as FieldOptionChain, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, d as ScoringMode, T as TextFieldOption, V as VectorIndexConfig, b as VectorQuantizationMode, g as VectorStorageMode, W as WriteOptions } from './schema-
|
|
1
|
+
import { e as StopWordOverride, C as CustomTokenizer } from './schema-CFnQ-WI3.js';
|
|
2
|
+
export { A as AnyDocument, c as BM25FieldParams, B as BM25Params, E as EmbeddingFieldConfig, f as FieldOptionChain, F as FieldType, I as IndexConfig, a as InsertOptions, P as PartitionConfig, S as SchemaDefinition, d as ScoringMode, T as TextFieldOption, V as VectorIndexConfig, b as VectorQuantizationMode, g as VectorStorageMode, W as WriteOptions } from './schema-CFnQ-WI3.js';
|
|
3
3
|
import { P as PartialWordSearch } from './partial-words-DTIHO8G4.js';
|
|
4
4
|
import { P as PatternSearch } from './pattern-Cledccbl.js';
|
|
5
5
|
import { S as SynonymSearch } from './synonyms-CpOAHPa-.js';
|
|
6
6
|
export { O as OneWaySynonymRule, a as SynonymRule, T as TwoWaySynonymRule } from './synonyms-CpOAHPa-.js';
|
|
7
7
|
import { L as LanguageModule } from './language-DB8KG1z5.js';
|
|
8
8
|
export { T as TokenizerConfig } from './language-DB8KG1z5.js';
|
|
9
|
-
import { s as Narsil } from './engine-
|
|
10
|
-
export { K as AdapterRegistrationOperations,
|
|
11
|
-
import { N as NarsilConfig } from './config-
|
|
12
|
-
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as MemoryConfig, d as NarsilPlugin, P as PartitionContext, Q as QuerySuggestionsConfig, R as RemoveContext, S as SearchContext, e as StaleAnalysis, U as UpdateContext, W as WorkerConfig, f as WorkerContext } from './config-
|
|
9
|
+
import { s as Narsil } from './engine-DGG_yX0p.js';
|
|
10
|
+
export { K as AdapterRegistrationOperations, A as AnalysisChange, O as AnalysisOperations, R as ArrayFilter, T as BM25Operations, B as BatchResult, U as ByteSource, C as Change, a as ChangePosition, b as ChangeType, c as ChangesBatch, d as ChangesEvent, e as ChangesOperations, f as ChangesResync, g as ChangesSetting, h as ChangesStatus, i as ChangesSubscribed, j as ClientErrorCode, k as ClientErrorCodes, W as ComparisonFilter, X as DocumentImportResult, Y as DocumentProjection, Z as DocumentWriteOperations, E as ErrorCode, l as ErrorCodes, _ as FacetConfig, $ as FacetResult, a0 as FieldFilter, a1 as FilterExpression, F as FollowChangesOptions, a2 as GeoFilter, a3 as GeoPolygonFilter, a4 as GeoRadiusFilter, J as GroupConfig, a5 as GroupReducer, G as GroupResult, a6 as HeldCheckpointReport, a7 as HighlightConfig, a8 as HighlightMatch, H as Hit, a9 as HybridConfig, I as IndexInfo, aa as IndexLifecycleOperations, z as IndexStats, L as ListParams, v as ListResult, ab as ListedDocument, M as MemoryStats, N as NarsilError, m as NarsilErrorCode, u as NarsilEventHandlers, D as NarsilEventMap, P as PartitionStatsResult, y as PreflightResult, ac as PresenceFilter, ad as ProcessMemoryReport, ae as QueryCoverage, Q as QueryParams, t as QueryResult, af as RerankConfig, ag as ScoreComponents, ah as SearchMode, S as ServerErrorCode, n as ServerErrorCodes, o as SnapshotOptions, ai as SortField, aj as SortSpec, ak as StringFilter, w as SuggestParams, x as SuggestResult, al as SyncOperations, p as SyncSetting, q as SyncStatus, am as SynonymOperations, an as TermMatchPolicy, ao as VectorMaintenanceOperations, V as VectorMaintenanceResult, ap as VectorQueryConfig, aq as WorkerCopyReport, r as isNarsilError } from './engine-DGG_yX0p.js';
|
|
11
|
+
import { N as NarsilConfig } from './config-B3-sw887.js';
|
|
12
|
+
export { A as AnalysisConfig, D as DurabilityConfig, I as IndexContext, a as IndexLifecycleConfig, b as InsertContext, M as MainCopyQueries, c as MemoryConfig, d as NarsilPlugin, P as PartitionContext, Q as QuerySuggestionsConfig, R as RemoveContext, S as SearchContext, e as StaleAnalysis, U as UpdateContext, W as WorkerConfig, f as WorkerContext } from './config-B3-sw887.js';
|
|
13
13
|
import { N as NativeStoredValues } from './stored-values-BWIMpiT_.js';
|
|
14
|
-
export { E as EmbeddingAdapter, I as InvalidationAdapter,
|
|
14
|
+
export { E as EmbeddingAdapter, a as IndexedDBLocation, I as InvalidationAdapter, b as InvalidationEvent, c as PartitionStatistics, P as PersistenceAdapter, R as RerankAdapter } from './adapters-8LcdNB77.js';
|
|
15
15
|
export { h as QuerySuggestionsBuildEvent, j as QuerySuggestionsWarningEvent } from './query-suggestions-CBYkrQ5T.js';
|
|
16
16
|
|
|
17
17
|
/**
|
package/dist/index.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './chunk-
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './chunk-WYH3QHBK.mjs';
|
|
2
2
|
import './chunk-2CKDCC65.mjs';
|
|
3
3
|
import './chunk-2AFZE7Y3.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-Y7GXGJFZ.mjs';
|
|
5
5
|
import './chunk-BUMREPLU.mjs';
|
|
6
6
|
import './chunk-7T4LAYT4.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-
|
|
7
|
+
import './chunk-4JLHHEBN.mjs';
|
|
8
|
+
import './chunk-EY5U4P5V.mjs';
|
|
9
9
|
import './chunk-U2BKYEC7.mjs';
|
|
10
10
|
import './chunk-FJXJZUWF.mjs';
|
|
11
11
|
import './chunk-5EI5CXO6.mjs';
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './react-native/chunks/chunk-
|
|
1
|
+
export { VERSION, clearNormalizationCache, configureNormalizationCache, createNarsil, getLanguage, getNormalizationCacheSize, getStopWords, getTokenizer, hasStopWords, hasTokenizer, isSimdAvailable, registerLanguage, registerPartialWordSearch, registerPatternSearch, registerStopWords, registerSynonyms, registerTokenizer, resetNormalizationCache } from './react-native/chunks/chunk-JEYZQA4Z.mjs';
|
|
2
2
|
import './react-native/chunks/chunk-HQEVQEDJ.mjs';
|
|
3
3
|
import './react-native/chunks/chunk-AKRYHAVG.mjs';
|
|
4
4
|
import './react-native/chunks/chunk-LX62Y4B5.mjs';
|
|
5
|
-
import './react-native/chunks/chunk-
|
|
5
|
+
import './react-native/chunks/chunk-ESVBC3AT.mjs';
|
|
6
6
|
import './react-native/chunks/chunk-UYKIV2KT.mjs';
|
|
7
7
|
import './react-native/chunks/chunk-CW2ZFXSE.mjs';
|
|
8
|
-
import './react-native/chunks/chunk-
|
|
9
|
-
import './react-native/chunks/chunk-
|
|
8
|
+
import './react-native/chunks/chunk-DN4FT62B.mjs';
|
|
9
|
+
import './react-native/chunks/chunk-AOLLV37A.mjs';
|
|
10
10
|
import './react-native/chunks/chunk-HRDMIQ63.mjs';
|
|
11
11
|
import './react-native/chunks/chunk-VUJCSQG5.mjs';
|
|
12
12
|
export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './react-native/chunks/chunk-43R4GCM7.mjs';
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import { createRelayClient, createThreadReadEngine, searchHandler, isLongQueryPortNote } from './chunk-
|
|
2
|
-
import { answerRequest } from './chunk-
|
|
3
|
-
import './chunk-
|
|
1
|
+
import { createRelayClient, createThreadReadEngine, searchHandler, isLongQueryPortNote } from './chunk-35RWLUVY.mjs';
|
|
2
|
+
import { answerRequest } from './chunk-ZTB7AWTW.mjs';
|
|
3
|
+
import './chunk-YGRJLI2Z.mjs';
|
|
4
4
|
import './chunk-TACP6VK6.mjs';
|
|
5
|
-
import './chunk-
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
5
|
+
import './chunk-ACNHJOGC.mjs';
|
|
6
|
+
import './chunk-LRIJ7U3T.mjs';
|
|
7
|
+
import './chunk-GTGQLCEI.mjs';
|
|
8
8
|
import './chunk-S5QMWECL.mjs';
|
|
9
9
|
import './chunk-LNGE7SLY.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
10
|
+
import './chunk-5SHWA6PE.mjs';
|
|
11
|
+
import './chunk-PQWK7KXS.mjs';
|
|
12
|
+
import './chunk-ZO7VLAIT.mjs';
|
|
13
13
|
import './chunk-R724DJSG.mjs';
|
|
14
14
|
import './chunk-V6EUBAIT.mjs';
|
|
15
|
-
import './chunk-
|
|
16
|
-
import './chunk-
|
|
15
|
+
import './chunk-G3KVEHIW.mjs';
|
|
16
|
+
import './chunk-WYH3QHBK.mjs';
|
|
17
17
|
import './chunk-2CKDCC65.mjs';
|
|
18
18
|
import './chunk-2AFZE7Y3.mjs';
|
|
19
|
-
import './chunk-
|
|
19
|
+
import './chunk-Y7GXGJFZ.mjs';
|
|
20
20
|
import './chunk-BUMREPLU.mjs';
|
|
21
21
|
import './chunk-7T4LAYT4.mjs';
|
|
22
|
-
import './chunk-
|
|
23
|
-
import './chunk-
|
|
22
|
+
import './chunk-4JLHHEBN.mjs';
|
|
23
|
+
import './chunk-EY5U4P5V.mjs';
|
|
24
24
|
import './chunk-U2BKYEC7.mjs';
|
|
25
25
|
import './chunk-FJXJZUWF.mjs';
|
|
26
26
|
import './chunk-5EI5CXO6.mjs';
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export { restoreVectorFields, setNestedValue } from './chunk-5SHWA6PE.mjs';
|
|
2
|
+
export { encodeDocument } from './chunk-2CKDCC65.mjs';
|
|
3
|
+
export { Decoder, MAP_TAGS, bindMsgpackHost, decode, decodeBoundedMessage, decodeMulti, headerAt } from './chunk-Y7GXGJFZ.mjs';
|
|
4
|
+
import './chunk-EY5U4P5V.mjs';
|
|
5
|
+
import './chunk-ZXX5GZEX.mjs';
|
|
6
|
+
import './chunk-4PY4JYMY.mjs';
|
|
7
|
+
import './chunk-ZKOQR7EA.mjs';
|
|
8
|
+
import './chunk-TGPKS2UM.mjs';
|
|
9
|
+
export { encode } from '@msgpack/msgpack';
|
|
@@ -1719,8 +1719,11 @@ function hitsKeptPerGroup(maxPerGroup) {
|
|
|
1719
1719
|
return Math.floor(maxPerGroup);
|
|
1720
1720
|
}
|
|
1721
1721
|
function applyGrouping(hits, group, documents) {
|
|
1722
|
+
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
1723
|
+
const limit = group.limit !== void 0 && Number.isFinite(group.limit) && group.limit >= 0 ? Math.min(Math.floor(group.limit), hits.length) : null;
|
|
1722
1724
|
if (!group.fields || group.fields.length === 0) {
|
|
1723
|
-
|
|
1725
|
+
if (hits.length === 0 || limit === 0) return [];
|
|
1726
|
+
return [reducedGroup({ values: {}, hits: hits.slice(0, keptPerGroup) }, group, hits, documents)];
|
|
1724
1727
|
}
|
|
1725
1728
|
if (hits.length > 4294967295)
|
|
1726
1729
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "The hit count exceeds the group layout");
|
|
@@ -1738,8 +1741,6 @@ function applyGrouping(hits, group, documents) {
|
|
|
1738
1741
|
if (at > 0 && hits[order[at - 1]].id !== hits[order[at]].id) rank++;
|
|
1739
1742
|
idRanks[order[at]] = rank;
|
|
1740
1743
|
}
|
|
1741
|
-
const keptPerGroup = hitsKeptPerGroup(group.maxPerGroup);
|
|
1742
|
-
const limit = group.limit !== void 0 && Number.isFinite(group.limit) && group.limit >= 0 ? Math.min(Math.floor(group.limit), hits.length) : null;
|
|
1743
1744
|
const heads = new Uint32Array(hits.length);
|
|
1744
1745
|
const offsets = new Uint32Array(hits.length + 1);
|
|
1745
1746
|
const positions = new Uint32Array(hits.length);
|
|
@@ -1764,23 +1765,24 @@ function applyGrouping(hits, group, documents) {
|
|
|
1764
1765
|
for (let field = 0; field < group.fields.length; field++) values[group.fields[field]] = row?.[field] ?? null;
|
|
1765
1766
|
const everyHit = [];
|
|
1766
1767
|
for (let member = offsets[at]; member < offsets[at + 1]; member++) everyHit.push(hits[positions[member]]);
|
|
1767
|
-
|
|
1768
|
-
if (group.reduce) {
|
|
1769
|
-
const folded = foldGroupReducer(
|
|
1770
|
-
group.reduce,
|
|
1771
|
-
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
1772
|
-
);
|
|
1773
|
-
if (folded.reducerError !== void 0) {
|
|
1774
|
-
result.reducerError = folded.reducerError;
|
|
1775
|
-
} else {
|
|
1776
|
-
result.reduced = folded.reduced;
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
groups.push(result);
|
|
1768
|
+
groups.push(reducedGroup({ values, hits: everyHit.slice(0, keptPerGroup) }, group, everyHit, documents));
|
|
1780
1769
|
}
|
|
1781
1770
|
if (limit !== null && groups.length > limit) groups.length = limit;
|
|
1782
1771
|
return groups;
|
|
1783
1772
|
}
|
|
1773
|
+
function reducedGroup(result, group, everyHit, documents) {
|
|
1774
|
+
if (!group.reduce) return result;
|
|
1775
|
+
const folded = foldGroupReducer(
|
|
1776
|
+
group.reduce,
|
|
1777
|
+
everyHit.map((hit) => ({ document: documents.document(hit.id), score: hit.score ?? 0 }))
|
|
1778
|
+
);
|
|
1779
|
+
if (folded.reducerError !== void 0) {
|
|
1780
|
+
result.reducerError = folded.reducerError;
|
|
1781
|
+
} else {
|
|
1782
|
+
result.reduced = folded.reduced;
|
|
1783
|
+
}
|
|
1784
|
+
return result;
|
|
1785
|
+
}
|
|
1784
1786
|
function foldGroupReducer(reduce, rows) {
|
|
1785
1787
|
try {
|
|
1786
1788
|
let accumulator = reduce.initialValue();
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
export { applyStoredIndexBm25, updateIndexBm25 } from './chunk-
|
|
2
|
-
import { readHeapStatistics, loadSchemaCodeFor, loadStorageCode, lookupOfStoredSynonyms, reconstructSchemaFromMetadata, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, prepareStoredSynonyms, getVectorFieldPaths, arrayPositionsAreStale, validatePartitionConfig, validateEmbeddingConfig } from './chunk-
|
|
3
|
-
export { bindOptionalWritesHost, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch } from './chunk-
|
|
1
|
+
export { applyStoredIndexBm25, updateIndexBm25 } from './chunk-DZMD4OMD.mjs';
|
|
2
|
+
import { readHeapStatistics, loadSchemaCodeFor, loadStorageCode, lookupOfStoredSynonyms, reconstructSchemaFromMetadata, getLanguage, validatePatternValueLimit, requireSurfaceFormsForPartialFields, prepareStoredSynonyms, getVectorFieldPaths, arrayPositionsAreStale, validatePartitionConfig, validateEmbeddingConfig } from './chunk-WYH3QHBK.mjs';
|
|
3
|
+
export { bindOptionalWritesHost, holdVectorsForRollback, insertDocument, insertDocumentBatch, prepareDocumentWrite, removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch } from './chunk-WYH3QHBK.mjs';
|
|
4
4
|
import './chunk-2CKDCC65.mjs';
|
|
5
5
|
import './chunk-2AFZE7Y3.mjs';
|
|
6
|
-
import './chunk-
|
|
6
|
+
import './chunk-Y7GXGJFZ.mjs';
|
|
7
7
|
import { isSyncEpoch, newSyncEpoch } from './chunk-BUMREPLU.mjs';
|
|
8
8
|
import './chunk-7T4LAYT4.mjs';
|
|
9
|
-
import './chunk-
|
|
10
|
-
import { validateSchema, validateVectorStorage, validateTextStorage } from './chunk-
|
|
9
|
+
import './chunk-4JLHHEBN.mjs';
|
|
10
|
+
import { validateSchema, validateVectorStorage, validateTextStorage } from './chunk-EY5U4P5V.mjs';
|
|
11
11
|
import './chunk-U2BKYEC7.mjs';
|
|
12
12
|
import './chunk-FJXJZUWF.mjs';
|
|
13
13
|
import './chunk-5EI5CXO6.mjs';
|
|
@@ -69,7 +69,7 @@ function epochStatesOfMetadata(metadata) {
|
|
|
69
69
|
|
|
70
70
|
// src/engine/index-from-metadata.ts
|
|
71
71
|
function isRecoveryConfigError(err) {
|
|
72
|
-
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
72
|
+
return err instanceof NarsilError && (err.code === ErrorCodes.CONFIG_INVALID || err.code === ErrorCodes.LANGUAGE_NOT_SUPPORTED || err.code === ErrorCodes.SCHEMA_INVALID_TYPE || err.code === ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION);
|
|
73
73
|
}
|
|
74
74
|
function resolveEmbeddingAdapter(deps, metadata, schema) {
|
|
75
75
|
const name = metadata.embedding?.adapter ?? null;
|
|
@@ -139,8 +139,9 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
139
139
|
return;
|
|
140
140
|
}
|
|
141
141
|
const indexConfig = reconstructSchemaFromMetadata(metadata);
|
|
142
|
-
|
|
142
|
+
let language;
|
|
143
143
|
try {
|
|
144
|
+
language = getLanguage(indexConfig.language ?? "english");
|
|
144
145
|
indexConfig.schema = validateSchema(indexConfig.schema);
|
|
145
146
|
validatePatternValueLimit(indexConfig.patternValueLimit);
|
|
146
147
|
requireSurfaceFormsForPartialFields(indexConfig.schema, indexConfig.surfaceForms);
|
|
@@ -1,26 +1,26 @@
|
|
|
1
|
-
import { resolvePartitionId } from './chunk-
|
|
2
|
-
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-
|
|
3
|
-
import './chunk-
|
|
1
|
+
import { resolvePartitionId } from './chunk-EZJ6HBG7.mjs';
|
|
2
|
+
import { readVectorFile, VECTOR_FILE_ENVELOPE_VERSION } from './chunk-FE2XILWN.mjs';
|
|
3
|
+
import './chunk-6LVQ3GLT.mjs';
|
|
4
4
|
import './chunk-M2Z5WVWW.mjs';
|
|
5
|
-
import { encodeSegmentManifest, loadFailed, directoryCheckpointStore, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-
|
|
5
|
+
import { encodeSegmentManifest, loadFailed, directoryCheckpointStore, decodeSections, SEGMENT_PAGE_BYTES, RECORD_PAGE_BYTES } from './chunk-H23FB267.mjs';
|
|
6
6
|
import { PARTITION_COPY_VECTOR_RUN_BYTES } from './chunk-JDDXV6UF.mjs';
|
|
7
|
-
import { segmentsPrefix, vectorFileKey } from './chunk-
|
|
7
|
+
import { segmentsPrefix, vectorFileKey } from './chunk-GTGQLCEI.mjs';
|
|
8
8
|
import './chunk-S5QMWECL.mjs';
|
|
9
9
|
import './chunk-LNGE7SLY.mjs';
|
|
10
|
-
import './chunk-
|
|
11
|
-
import './chunk-
|
|
12
|
-
import './chunk-
|
|
10
|
+
import './chunk-5SHWA6PE.mjs';
|
|
11
|
+
import './chunk-PQWK7KXS.mjs';
|
|
12
|
+
import './chunk-ZO7VLAIT.mjs';
|
|
13
13
|
import { vectorsPerFile, isDead, VECTOR_FILE_PAYLOAD_VERSION } from './chunk-R724DJSG.mjs';
|
|
14
14
|
import './chunk-V6EUBAIT.mjs';
|
|
15
|
-
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-
|
|
16
|
-
import { generateId, isCompositePartition } from './chunk-
|
|
15
|
+
import { HEADER_SIZE, crc32Init, crc32Final, engineVersionParts, writeHeader, RESERVED_SIZE, crc32Update } from './chunk-G3KVEHIW.mjs';
|
|
16
|
+
import { generateId, isCompositePartition } from './chunk-WYH3QHBK.mjs';
|
|
17
17
|
import './chunk-2CKDCC65.mjs';
|
|
18
18
|
import './chunk-2AFZE7Y3.mjs';
|
|
19
|
-
import { decode } from './chunk-
|
|
19
|
+
import { decode } from './chunk-Y7GXGJFZ.mjs';
|
|
20
20
|
import './chunk-BUMREPLU.mjs';
|
|
21
21
|
import './chunk-7T4LAYT4.mjs';
|
|
22
|
-
import './chunk-
|
|
23
|
-
import { parseVectorDimension } from './chunk-
|
|
22
|
+
import './chunk-4JLHHEBN.mjs';
|
|
23
|
+
import { parseVectorDimension } from './chunk-EY5U4P5V.mjs';
|
|
24
24
|
import './chunk-U2BKYEC7.mjs';
|
|
25
25
|
import './chunk-FJXJZUWF.mjs';
|
|
26
26
|
import './chunk-5EI5CXO6.mjs';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { spellingCountsOf, encodePartialSpellings, countSpelling, SurfaceTable, SURFACE_TABLE_START_CAPACITY, StringPlan, SegmentRefusal, reserveStrings, grownCapacity, PARTIAL_WORD_RUN_CODE_POINTS, encodeStringBlob, sameFields, countDocumentSpellings, tokenize, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GROWING_PARTIAL_PENDING_ENTRY_BYTES, RUN_TABLE_START_CAPACITY, TERM_REGION_GROWTH_FACTOR, RUN_MEMBER_START_CAPACITY, SLICE_CAPACITIES, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, PoolPlan, requireRoom, releaseRange, requireSegment, termText, partitionDocumentFrequency, visitDocumentSpellings, visitDocumentSurfaceSpellings, DEFAULT_FUZZY_PREFIX_LENGTH, resolveStopWords, transformToken, expansionSource, liveFrequency, codePointCountOf, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, FNV_OFFSET_BASIS, typedRegion, regionsOverlap, mergeText, FNV_PRIME } from './browser/chunk-
|
|
2
|
-
import { fuzzyTermMatches, storedText } from './browser/chunk-
|
|
1
|
+
import { spellingCountsOf, encodePartialSpellings, countSpelling, SurfaceTable, SURFACE_TABLE_START_CAPACITY, StringPlan, SegmentRefusal, reserveStrings, grownCapacity, PARTIAL_WORD_RUN_CODE_POINTS, encodeStringBlob, sameFields, countDocumentSpellings, tokenize, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, GROWING_PARTIAL_ENTRY_BYTES, GROWING_PARTIAL_RUN_BYTES, GROWING_PARTIAL_RUN_MEMBER_BYTES, GROWING_PARTIAL_PENDING_ENTRY_BYTES, RUN_TABLE_START_CAPACITY, TERM_REGION_GROWTH_FACTOR, RUN_MEMBER_START_CAPACITY, SLICE_CAPACITIES, TERM_SLOT_LOAD_DENOMINATOR, TERM_SLOT_LOAD_NUMERATOR, PoolPlan, requireRoom, releaseRange, requireSegment, termText, partitionDocumentFrequency, visitDocumentSpellings, visitDocumentSurfaceSpellings, DEFAULT_FUZZY_PREFIX_LENGTH, resolveStopWords, transformToken, expansionSource, liveFrequency, codePointCountOf, GROWING_PARTIAL_PENDING_ENTRIES_PER_ENTRY, FNV_OFFSET_BASIS, typedRegion, regionsOverlap, mergeText, FNV_PRIME } from './browser/chunk-EKISQBV2.mjs';
|
|
2
|
+
import { fuzzyTermMatches, storedText } from './browser/chunk-56UNHLAF.mjs';
|
|
3
3
|
import './browser/chunk-YJHGOJAC.mjs';
|
|
4
|
-
import { compareCodePoints, codePointCount, NarsilError, ErrorCodes } from './browser/chunk-
|
|
4
|
+
import { compareCodePoints, codePointCount, NarsilError, ErrorCodes } from './browser/chunk-VAPCFSMX.mjs';
|
|
5
5
|
|
|
6
6
|
// src/core/expansion/partial-bindings.ts
|
|
7
7
|
var PARTIAL_BUDGET;
|
package/dist/partial-words.mjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import { partialWordSearchModule } from './chunk-
|
|
1
|
+
import { partialWordSearchModule } from './chunk-WYH3QHBK.mjs';
|
|
2
2
|
import './chunk-2CKDCC65.mjs';
|
|
3
3
|
import './chunk-2AFZE7Y3.mjs';
|
|
4
|
-
import './chunk-
|
|
4
|
+
import './chunk-Y7GXGJFZ.mjs';
|
|
5
5
|
import './chunk-BUMREPLU.mjs';
|
|
6
6
|
import './chunk-7T4LAYT4.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-
|
|
7
|
+
import './chunk-4JLHHEBN.mjs';
|
|
8
|
+
import './chunk-EY5U4P5V.mjs';
|
|
9
9
|
import './chunk-U2BKYEC7.mjs';
|
|
10
10
|
import './chunk-FJXJZUWF.mjs';
|
|
11
11
|
import './chunk-5EI5CXO6.mjs';
|
package/dist/pattern.browser.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { storedValuesOf, bitsetHas, bitsetSet, GrowableUint32, PATTERN_RUN_CODE_POINTS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, createBitSet, MAX_PATTERN_POSITIONS, MATCHER_STATE_OVERHEAD_BYTES, MAX_PATTERN_TOLERANCE, bitsetAnd, bitsetNot, PATTERN_EXPRESSION_OPERATORS, invalid, MAX_PATTERN_TEXT_CODE_POINTS, TYPO_CLAIM_SLOTS_PER_WORD, storedText, MATCHER_CODE_POINT_BYTES, MATCHER_SIMULATION_BYTES_PER_POSITION, tabulatedLevenshtein } from './browser/chunk-
|
|
2
|
-
import { NarsilError, ErrorCodes, codePointCount, exceedsCodePoints } from './browser/chunk-
|
|
1
|
+
import { storedValuesOf, bitsetHas, bitsetSet, GrowableUint32, PATTERN_RUN_CODE_POINTS, PATTERN_TEST_OPERATORS, PATTERN_ELEMENT_OPERATORS, createBitSet, MAX_PATTERN_POSITIONS, MATCHER_STATE_OVERHEAD_BYTES, MAX_PATTERN_TOLERANCE, bitsetAnd, bitsetNot, PATTERN_EXPRESSION_OPERATORS, invalid, MAX_PATTERN_TEXT_CODE_POINTS, TYPO_CLAIM_SLOTS_PER_WORD, storedText, MATCHER_CODE_POINT_BYTES, MATCHER_SIMULATION_BYTES_PER_POSITION, tabulatedLevenshtein } from './browser/chunk-56UNHLAF.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes, codePointCount, exceedsCodePoints } from './browser/chunk-VAPCFSMX.mjs';
|
|
3
3
|
|
|
4
4
|
// src/pattern/constants.ts
|
|
5
5
|
var MATCHER_TRANSITION_BYTES = 48;
|
package/dist/pattern.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { patternSearchModule } from './chunk-
|
|
2
|
-
import './chunk-
|
|
1
|
+
import { patternSearchModule } from './chunk-4JLHHEBN.mjs';
|
|
2
|
+
import './chunk-EY5U4P5V.mjs';
|
|
3
3
|
import './chunk-ZXX5GZEX.mjs';
|
|
4
4
|
import './chunk-4PY4JYMY.mjs';
|
|
5
5
|
import './chunk-ZKOQR7EA.mjs';
|