@delali/narsil 0.1.15 → 0.2.1-next.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +54 -0
- package/README.md +38 -982
- package/dist/adapters/filesystem.d.ts +23 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +25 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +14 -1
- package/dist/adapters-CVqI5NBt.d.ts +149 -0
- package/dist/{chunk-HMO6OMZ3.mjs → chunk-ASI6AOW2.mjs} +1073 -506
- package/dist/{chunk-HRABJLPB.mjs → chunk-BKAI2WXI.mjs} +1 -0
- package/dist/{chunk-WISGW24C.mjs → chunk-FKK76OR7.mjs} +56 -55
- package/dist/chunk-GHJWC2E5.mjs +9 -0
- package/dist/chunk-KH3YGMVT.mjs +15 -0
- package/dist/chunk-TGP5M5R7.mjs +880 -0
- package/dist/chunk-U2BKYEC7.mjs +254 -0
- package/dist/{chunk-ALB35PRQ.mjs → chunk-UNJZ7ZDB.mjs} +18 -0
- package/dist/{chunk-MEW7HD75.mjs → chunk-WLR5BFL3.mjs} +2557 -1166
- package/dist/config-CAjTHzkH.d.ts +256 -0
- package/dist/direct-executor-3W4QRA4E.mjs +5 -0
- package/dist/distribution/coordinator/etcd.d.ts +44 -3
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +14 -3
- package/dist/distribution.d.ts +318 -45
- package/dist/distribution.mjs +1875 -1772
- package/dist/embeddings/openai.d.ts +28 -1
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/engine-CCrlfLyq.d.ts +431 -0
- package/dist/index.browser.mjs +4571 -1776
- package/dist/index.d.ts +180 -6
- package/dist/index.mjs +6 -5
- package/dist/invalidation/broadcast-channel.d.ts +20 -1
- package/dist/invalidation/filesystem.d.ts +24 -1
- package/dist/invalidation/noop.d.ts +14 -1
- package/dist/language-DB8KG1z5.d.ts +53 -0
- package/dist/languages/albanian.d.ts +15 -0
- package/dist/languages/albanian.mjs +103 -0
- package/dist/languages/amharic.d.ts +15 -0
- package/dist/languages/amharic.mjs +240 -0
- package/dist/languages/arabic.d.ts +10 -1
- package/dist/languages/arabic.mjs +939 -154
- package/dist/languages/armenian.d.ts +10 -1
- package/dist/languages/armenian.mjs +275 -243
- package/dist/languages/azerbaijani.d.ts +15 -0
- package/dist/languages/azerbaijani.mjs +152 -0
- package/dist/languages/bambara.d.ts +15 -0
- package/dist/languages/bambara.mjs +60 -0
- package/dist/languages/basque.d.ts +14 -0
- package/dist/languages/basque.mjs +726 -0
- package/dist/languages/belarusian.d.ts +15 -0
- package/dist/languages/belarusian.mjs +100 -0
- package/dist/languages/bengali.d.ts +15 -0
- package/dist/languages/bengali.mjs +128 -0
- package/dist/languages/bosnian.d.ts +15 -0
- package/dist/languages/bosnian.mjs +197 -0
- package/dist/languages/breton.d.ts +15 -0
- package/dist/languages/breton.mjs +96 -0
- package/dist/languages/bulgarian.d.ts +10 -1
- package/dist/languages/bulgarian.mjs +11 -2
- package/dist/languages/burmese.d.ts +15 -0
- package/dist/languages/burmese.mjs +57 -0
- package/dist/languages/catalan.d.ts +14 -0
- package/dist/languages/catalan.mjs +987 -0
- package/dist/languages/chinese.d.ts +11 -1
- package/dist/languages/chinese.mjs +4 -3
- package/dist/languages/croatian.d.ts +15 -0
- package/dist/languages/croatian.mjs +191 -0
- package/dist/languages/czech.d.ts +14 -0
- package/dist/languages/czech.mjs +628 -0
- package/dist/languages/dagbani.d.ts +11 -1
- package/dist/languages/dagbani.mjs +11 -2
- package/dist/languages/danish.d.ts +10 -1
- package/dist/languages/danish.mjs +215 -110
- package/dist/languages/dutch.d.ts +10 -1
- package/dist/languages/dutch.mjs +1248 -163
- package/dist/languages/english.d.ts +10 -1
- package/dist/languages/english.mjs +2 -1
- package/dist/languages/esperanto.d.ts +14 -0
- package/dist/languages/esperanto.mjs +689 -0
- package/dist/languages/estonian.d.ts +14 -0
- package/dist/languages/estonian.mjs +2160 -0
- package/dist/languages/ewe.d.ts +11 -1
- package/dist/languages/ewe.mjs +11 -2
- package/dist/languages/faroese.d.ts +15 -0
- package/dist/languages/faroese.mjs +93 -0
- package/dist/languages/fijian.d.ts +15 -0
- package/dist/languages/fijian.mjs +55 -0
- package/dist/languages/finnish.d.ts +10 -1
- package/dist/languages/finnish.mjs +528 -263
- package/dist/languages/french.d.ts +10 -1
- package/dist/languages/french.mjs +950 -437
- package/dist/languages/ga.d.ts +11 -1
- package/dist/languages/ga.mjs +2 -1
- package/dist/languages/galician.d.ts +15 -0
- package/dist/languages/galician.mjs +172 -0
- package/dist/languages/georgian.d.ts +15 -0
- package/dist/languages/georgian.mjs +91 -0
- package/dist/languages/german.d.ts +10 -1
- package/dist/languages/german.mjs +432 -162
- package/dist/languages/greek.d.ts +10 -1
- package/dist/languages/greek.mjs +1970 -281
- package/dist/languages/guarani.d.ts +15 -0
- package/dist/languages/guarani.mjs +132 -0
- package/dist/languages/gujarati.d.ts +15 -0
- package/dist/languages/gujarati.mjs +234 -0
- package/dist/languages/haitian-creole.d.ts +15 -0
- package/dist/languages/haitian-creole.mjs +106 -0
- package/dist/languages/hausa.d.ts +11 -1
- package/dist/languages/hausa.mjs +2 -1
- package/dist/languages/hawaiian.d.ts +15 -0
- package/dist/languages/hawaiian.mjs +69 -0
- package/dist/languages/hebrew.d.ts +15 -0
- package/dist/languages/hebrew.mjs +213 -0
- package/dist/languages/hindi.d.ts +10 -1
- package/dist/languages/hindi.mjs +175 -97
- package/dist/languages/hungarian.d.ts +10 -1
- package/dist/languages/hungarian.mjs +412 -326
- package/dist/languages/icelandic.d.ts +15 -0
- package/dist/languages/icelandic.mjs +161 -0
- package/dist/languages/igbo.d.ts +11 -1
- package/dist/languages/igbo.mjs +2 -1
- package/dist/languages/indonesian.d.ts +10 -1
- package/dist/languages/indonesian.mjs +329 -137
- package/dist/languages/irish.d.ts +10 -1
- package/dist/languages/irish.mjs +223 -139
- package/dist/languages/italian.d.ts +10 -1
- package/dist/languages/italian.mjs +703 -406
- package/dist/languages/japanese.d.ts +11 -1
- package/dist/languages/japanese.mjs +5 -108
- package/dist/languages/kannada.d.ts +15 -0
- package/dist/languages/kannada.mjs +94 -0
- package/dist/languages/kazakh.d.ts +15 -0
- package/dist/languages/kazakh.mjs +90 -0
- package/dist/languages/khmer.d.ts +15 -0
- package/dist/languages/khmer.mjs +181 -0
- package/dist/languages/kinyarwanda.d.ts +15 -0
- package/dist/languages/kinyarwanda.mjs +99 -0
- package/dist/languages/kirundi.d.ts +15 -0
- package/dist/languages/kirundi.mjs +92 -0
- package/dist/languages/korean.d.ts +15 -0
- package/dist/languages/korean.mjs +252 -0
- package/dist/languages/kurmanji.d.ts +15 -0
- package/dist/languages/kurmanji.mjs +50 -0
- package/dist/languages/kyrgyz.d.ts +15 -0
- package/dist/languages/kyrgyz.mjs +108 -0
- package/dist/languages/lao.d.ts +15 -0
- package/dist/languages/lao.mjs +63 -0
- package/dist/languages/latin.d.ts +15 -0
- package/dist/languages/latin.mjs +69 -0
- package/dist/languages/latvian.d.ts +15 -0
- package/dist/languages/latvian.mjs +173 -0
- package/dist/languages/lingala.d.ts +15 -0
- package/dist/languages/lingala.mjs +61 -0
- package/dist/languages/lithuanian.d.ts +14 -0
- package/dist/languages/lithuanian.mjs +571 -0
- package/dist/languages/luxembourgish.d.ts +15 -0
- package/dist/languages/luxembourgish.mjs +214 -0
- package/dist/languages/macedonian.d.ts +15 -0
- package/dist/languages/macedonian.mjs +808 -0
- package/dist/languages/malagasy.d.ts +15 -0
- package/dist/languages/malagasy.mjs +67 -0
- package/dist/languages/malay.d.ts +15 -0
- package/dist/languages/malay.mjs +482 -0
- package/dist/languages/malayalam.d.ts +15 -0
- package/dist/languages/malayalam.mjs +122 -0
- package/dist/languages/maltese.d.ts +15 -0
- package/dist/languages/maltese.mjs +139 -0
- package/dist/languages/maori.d.ts +15 -0
- package/dist/languages/maori.mjs +79 -0
- package/dist/languages/marathi.d.ts +15 -0
- package/dist/languages/marathi.mjs +111 -0
- package/dist/languages/nepali.d.ts +10 -1
- package/dist/languages/nepali.mjs +209 -99
- package/dist/languages/norwegian.d.ts +10 -1
- package/dist/languages/norwegian.mjs +218 -85
- package/dist/languages/oromo.d.ts +15 -0
- package/dist/languages/oromo.mjs +78 -0
- package/dist/languages/persian.d.ts +14 -0
- package/dist/languages/persian.mjs +710 -0
- package/dist/languages/polish.d.ts +14 -0
- package/dist/languages/polish.mjs +614 -0
- package/dist/languages/portuguese.d.ts +10 -1
- package/dist/languages/portuguese.mjs +613 -380
- package/dist/languages/punjabi.d.ts +15 -0
- package/dist/languages/punjabi.mjs +197 -0
- package/dist/languages/romanian.d.ts +10 -1
- package/dist/languages/romanian.mjs +561 -382
- package/dist/languages/russian.d.ts +10 -1
- package/dist/languages/russian.mjs +373 -244
- package/dist/languages/samoan.d.ts +15 -0
- package/dist/languages/samoan.mjs +63 -0
- package/dist/languages/sanskrit.d.ts +10 -1
- package/dist/languages/sanskrit.mjs +2 -2
- package/dist/languages/scottish-gaelic.d.ts +15 -0
- package/dist/languages/scottish-gaelic.mjs +370 -0
- package/dist/languages/serbian.d.ts +10 -1
- package/dist/languages/serbian.mjs +3635 -155
- package/dist/languages/shona.d.ts +15 -0
- package/dist/languages/shona.mjs +91 -0
- package/dist/languages/sinhala.d.ts +15 -0
- package/dist/languages/sinhala.mjs +182 -0
- package/dist/languages/slovak.d.ts +15 -0
- package/dist/languages/slovak.mjs +430 -0
- package/dist/languages/slovenian.d.ts +10 -1
- package/dist/languages/slovenian.mjs +11 -2
- package/dist/languages/sorani.d.ts +15 -0
- package/dist/languages/sorani.mjs +156 -0
- package/dist/languages/spanish.d.ts +10 -1
- package/dist/languages/spanish.mjs +676 -374
- package/dist/languages/swahili.d.ts +10 -1
- package/dist/languages/swahili.mjs +2 -1
- package/dist/languages/swedish.d.ts +10 -1
- package/dist/languages/swedish.mjs +237 -95
- package/dist/languages/tagalog.d.ts +15 -0
- package/dist/languages/tagalog.mjs +159 -0
- package/dist/languages/tamil.d.ts +10 -1
- package/dist/languages/tamil.mjs +836 -64
- package/dist/languages/tatar.d.ts +15 -0
- package/dist/languages/tatar.mjs +1017 -0
- package/dist/languages/telugu.d.ts +15 -0
- package/dist/languages/telugu.mjs +62 -0
- package/dist/languages/thai.d.ts +15 -0
- package/dist/languages/thai.mjs +65 -0
- package/dist/languages/tibetan.d.ts +15 -0
- package/dist/languages/tibetan.mjs +119 -0
- package/dist/languages/tigrinya.d.ts +15 -0
- package/dist/languages/tigrinya.mjs +194 -0
- package/dist/languages/tongan.d.ts +15 -0
- package/dist/languages/tongan.mjs +56 -0
- package/dist/languages/turkish.d.ts +10 -1
- package/dist/languages/turkish.mjs +1246 -393
- package/dist/languages/twi.d.ts +11 -1
- package/dist/languages/twi.mjs +13 -2
- package/dist/languages/ukrainian.d.ts +10 -1
- package/dist/languages/ukrainian.mjs +125 -47
- package/dist/languages/urdu.d.ts +15 -0
- package/dist/languages/urdu.mjs +528 -0
- package/dist/languages/vietnamese.d.ts +15 -0
- package/dist/languages/vietnamese.mjs +277 -0
- package/dist/languages/welsh.d.ts +15 -0
- package/dist/languages/welsh.mjs +446 -0
- package/dist/languages/wolof.d.ts +15 -0
- package/dist/languages/wolof.mjs +79 -0
- package/dist/languages/xhosa.d.ts +15 -0
- package/dist/languages/xhosa.mjs +95 -0
- package/dist/languages/yoruba.d.ts +11 -1
- package/dist/languages/yoruba.mjs +1 -0
- package/dist/languages/zulu.d.ts +11 -1
- package/dist/languages/zulu.mjs +2 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +13 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +6 -5
- package/dist/schema-CE6S_aAr.d.ts +208 -0
- package/dist/search-CgQQIciB.d.ts +786 -0
- package/dist/server.d.ts +214 -12
- package/dist/server.mjs +41 -2
- package/dist/types-BV3nAKTL.d.ts +302 -0
- package/dist/vector/hnsw-build-worker.d.ts +68 -2
- package/dist/vector/hnsw-build-worker.mjs +0 -2
- package/dist/workers/entry.mjs +2 -2
- package/package.json +14 -4
- package/dist/adapters-Bo2JQg8b.d.ts +0 -37
- package/dist/chunk-TW6BXTOL.mjs +0 -338
- package/dist/config-CojEPOgS.d.ts +0 -481
- package/dist/direct-executor-E52FFGDE.mjs +0 -4
- package/dist/language-ONQ3iVDf.d.ts +0 -14
- package/dist/narsil-D-84Nvm4.d.ts +0 -72
- package/dist/schema-DCgkO1j4.d.ts +0 -65
- package/dist/types-DHX6zGeD.d.ts +0 -73
|
@@ -1,6 +1,42 @@
|
|
|
1
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-UNJZ7ZDB.mjs';
|
|
2
2
|
import { decode, encode } from '@msgpack/msgpack';
|
|
3
3
|
|
|
4
|
+
// src/distribution/coordinator/etcd/types.ts
|
|
5
|
+
var DEFAULT_ETCD_CONFIG = {
|
|
6
|
+
endpoints: ["http://localhost:2379"],
|
|
7
|
+
keyPrefix: "_narsil",
|
|
8
|
+
nodeHeartbeatTtlSeconds: 30,
|
|
9
|
+
leaseTtlSeconds: 15
|
|
10
|
+
};
|
|
11
|
+
var ETCD_KEY_NODES = "nodes";
|
|
12
|
+
var ETCD_KEY_ALLOCATION = "allocation";
|
|
13
|
+
var ETCD_KEY_PARTITION = "partition";
|
|
14
|
+
var ETCD_KEY_SCHEMA = "schema";
|
|
15
|
+
function buildKey(prefix, ...segments) {
|
|
16
|
+
return [prefix, ...segments].join("/");
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
// src/distribution/coordinator/etcd/keys.ts
|
|
20
|
+
var ETCD_KEY_LEASE = "lease";
|
|
21
|
+
var ETCD_KEY_GENERIC = "kv";
|
|
22
|
+
function coordinatorKeys(config) {
|
|
23
|
+
const prefix = config.keyPrefix;
|
|
24
|
+
return {
|
|
25
|
+
node: (nodeId) => buildKey(prefix, ETCD_KEY_NODES, nodeId),
|
|
26
|
+
nodePrefix: () => buildKey(prefix, ETCD_KEY_NODES),
|
|
27
|
+
allocation: (indexName) => buildKey(prefix, ETCD_KEY_ALLOCATION, indexName),
|
|
28
|
+
allocationPrefix: () => buildKey(prefix, ETCD_KEY_ALLOCATION),
|
|
29
|
+
partition: (indexName, partitionId) => buildKey(prefix, ETCD_KEY_PARTITION, indexName, String(partitionId)),
|
|
30
|
+
schema: (indexName) => buildKey(prefix, ETCD_KEY_SCHEMA, indexName),
|
|
31
|
+
schemaPrefix: () => buildKey(prefix, ETCD_KEY_SCHEMA),
|
|
32
|
+
lease: (key) => buildKey(prefix, ETCD_KEY_LEASE, key),
|
|
33
|
+
generic: (key) => buildKey(prefix, ETCD_KEY_GENERIC, key)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
function extractSuffix(fullKey, prefix) {
|
|
37
|
+
return fullKey.slice(prefix.length + 1);
|
|
38
|
+
}
|
|
39
|
+
|
|
4
40
|
// src/distribution/coordinator/etcd/leases.ts
|
|
5
41
|
var LeaseManager = class {
|
|
6
42
|
leases = /* @__PURE__ */ new Map();
|
|
@@ -99,21 +135,6 @@ function deserializeSchema(data) {
|
|
|
99
135
|
return decode(data);
|
|
100
136
|
}
|
|
101
137
|
|
|
102
|
-
// src/distribution/coordinator/etcd/types.ts
|
|
103
|
-
var DEFAULT_ETCD_CONFIG = {
|
|
104
|
-
endpoints: ["http://localhost:2379"],
|
|
105
|
-
keyPrefix: "_narsil",
|
|
106
|
-
nodeHeartbeatTtlSeconds: 30,
|
|
107
|
-
leaseTtlSeconds: 15
|
|
108
|
-
};
|
|
109
|
-
var ETCD_KEY_NODES = "nodes";
|
|
110
|
-
var ETCD_KEY_ALLOCATION = "allocation";
|
|
111
|
-
var ETCD_KEY_PARTITION = "partition";
|
|
112
|
-
var ETCD_KEY_SCHEMA = "schema";
|
|
113
|
-
function buildKey(prefix, ...segments) {
|
|
114
|
-
return [prefix, ...segments].join("/");
|
|
115
|
-
}
|
|
116
|
-
|
|
117
138
|
// src/distribution/coordinator/etcd/validation.ts
|
|
118
139
|
var MAX_NODE_ID_LENGTH = 255;
|
|
119
140
|
var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
|
|
@@ -164,27 +185,7 @@ async function createEtcdCoordinator(config) {
|
|
|
164
185
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
|
|
165
186
|
}
|
|
166
187
|
}
|
|
167
|
-
|
|
168
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_NODES, nodeId);
|
|
169
|
-
}
|
|
170
|
-
function allocationKey(indexName) {
|
|
171
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_ALLOCATION, indexName);
|
|
172
|
-
}
|
|
173
|
-
function partitionKey(indexName, partitionId) {
|
|
174
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_PARTITION, indexName, String(partitionId));
|
|
175
|
-
}
|
|
176
|
-
function schemaKey(indexName) {
|
|
177
|
-
return buildKey(resolvedConfig.keyPrefix, ETCD_KEY_SCHEMA, indexName);
|
|
178
|
-
}
|
|
179
|
-
function leaseKey(key) {
|
|
180
|
-
return buildKey(resolvedConfig.keyPrefix, "lease", key);
|
|
181
|
-
}
|
|
182
|
-
function genericKey(key) {
|
|
183
|
-
return buildKey(resolvedConfig.keyPrefix, "kv", key);
|
|
184
|
-
}
|
|
185
|
-
function extractSuffix(fullKey, prefix) {
|
|
186
|
-
return fullKey.slice(prefix.length + 1);
|
|
187
|
-
}
|
|
188
|
+
const keys = coordinatorKeys(resolvedConfig);
|
|
188
189
|
function addWatcher(watcher) {
|
|
189
190
|
if (watchers.size >= MAX_WATCHERS) {
|
|
190
191
|
watcher.cancel().catch(() => {
|
|
@@ -207,7 +208,7 @@ async function createEtcdCoordinator(config) {
|
|
|
207
208
|
const ttl = resolvedConfig.nodeHeartbeatTtlSeconds;
|
|
208
209
|
const lease = client.lease(ttl);
|
|
209
210
|
await lease.grant();
|
|
210
|
-
const key =
|
|
211
|
+
const key = keys.node(registration.nodeId);
|
|
211
212
|
const data = Buffer.from(serializeNodeRegistration(registration));
|
|
212
213
|
await lease.put(key).value(data).exec();
|
|
213
214
|
const existing = leaseManager.get(key);
|
|
@@ -220,7 +221,7 @@ async function createEtcdCoordinator(config) {
|
|
|
220
221
|
async deregisterNode(nodeId) {
|
|
221
222
|
assertNotShutdown();
|
|
222
223
|
validateNodeId(nodeId);
|
|
223
|
-
const key =
|
|
224
|
+
const key = keys.node(nodeId);
|
|
224
225
|
const entry = leaseManager.remove(key);
|
|
225
226
|
if (entry !== void 0) {
|
|
226
227
|
await entry.lease.revoke().catch(() => {
|
|
@@ -230,7 +231,7 @@ async function createEtcdCoordinator(config) {
|
|
|
230
231
|
},
|
|
231
232
|
async listNodes() {
|
|
232
233
|
assertNotShutdown();
|
|
233
|
-
const prefix =
|
|
234
|
+
const prefix = keys.nodePrefix();
|
|
234
235
|
const result = await client.getAll().prefix(prefix).buffers();
|
|
235
236
|
const nodes = [];
|
|
236
237
|
for (const key of Object.keys(result)) {
|
|
@@ -243,7 +244,7 @@ async function createEtcdCoordinator(config) {
|
|
|
243
244
|
},
|
|
244
245
|
async watchNodes(handler) {
|
|
245
246
|
assertNotShutdown();
|
|
246
|
-
const prefix =
|
|
247
|
+
const prefix = keys.nodePrefix();
|
|
247
248
|
const watcher = await client.watch().prefix(prefix).create();
|
|
248
249
|
addWatcher(watcher);
|
|
249
250
|
watcher.on("put", (kv) => {
|
|
@@ -262,7 +263,7 @@ async function createEtcdCoordinator(config) {
|
|
|
262
263
|
},
|
|
263
264
|
async getAllocation(indexName) {
|
|
264
265
|
assertNotShutdown();
|
|
265
|
-
const key =
|
|
266
|
+
const key = keys.allocation(indexName);
|
|
266
267
|
const buf = await client.get(key).buffer();
|
|
267
268
|
if (buf === null) {
|
|
268
269
|
return null;
|
|
@@ -271,7 +272,7 @@ async function createEtcdCoordinator(config) {
|
|
|
271
272
|
},
|
|
272
273
|
async putAllocation(indexName, table, expectedVersion) {
|
|
273
274
|
assertNotShutdown();
|
|
274
|
-
const key =
|
|
275
|
+
const key = keys.allocation(indexName);
|
|
275
276
|
const data = Buffer.from(serializeAllocationTable(table));
|
|
276
277
|
if (expectedVersion === void 0) {
|
|
277
278
|
await client.put(key).value(data).exec();
|
|
@@ -295,7 +296,7 @@ async function createEtcdCoordinator(config) {
|
|
|
295
296
|
},
|
|
296
297
|
async watchAllocation(handler) {
|
|
297
298
|
assertNotShutdown();
|
|
298
|
-
const prefix =
|
|
299
|
+
const prefix = keys.allocationPrefix();
|
|
299
300
|
const watcher = await client.watch().prefix(prefix).create();
|
|
300
301
|
addWatcher(watcher);
|
|
301
302
|
watcher.on("put", (kv) => {
|
|
@@ -309,7 +310,7 @@ async function createEtcdCoordinator(config) {
|
|
|
309
310
|
},
|
|
310
311
|
async getPartitionState(indexName, partitionId) {
|
|
311
312
|
assertNotShutdown();
|
|
312
|
-
const key =
|
|
313
|
+
const key = keys.partition(indexName, partitionId);
|
|
313
314
|
const value = await client.get(key).string();
|
|
314
315
|
if (value === null) {
|
|
315
316
|
return "UNASSIGNED";
|
|
@@ -318,12 +319,12 @@ async function createEtcdCoordinator(config) {
|
|
|
318
319
|
},
|
|
319
320
|
async putPartitionState(indexName, partitionId, state) {
|
|
320
321
|
assertNotShutdown();
|
|
321
|
-
const key =
|
|
322
|
+
const key = keys.partition(indexName, partitionId);
|
|
322
323
|
await client.put(key).value(state).exec();
|
|
323
324
|
},
|
|
324
325
|
async acquireLease(key, nodeId, ttlMs) {
|
|
325
326
|
assertNotShutdown();
|
|
326
|
-
const etcdKey =
|
|
327
|
+
const etcdKey = keys.lease(key);
|
|
327
328
|
const ttlSeconds = Math.max(1, Math.ceil(ttlMs / 1e3));
|
|
328
329
|
const existing = leaseManager.get(etcdKey);
|
|
329
330
|
if (existing !== void 0 && existing.nodeId === nodeId) {
|
|
@@ -348,7 +349,7 @@ async function createEtcdCoordinator(config) {
|
|
|
348
349
|
},
|
|
349
350
|
async renewLease(key, nodeId, _ttlMs) {
|
|
350
351
|
assertNotShutdown();
|
|
351
|
-
const etcdKey =
|
|
352
|
+
const etcdKey = keys.lease(key);
|
|
352
353
|
const entry = leaseManager.getByNodeId(etcdKey, nodeId);
|
|
353
354
|
if (entry === void 0) {
|
|
354
355
|
return false;
|
|
@@ -363,7 +364,7 @@ async function createEtcdCoordinator(config) {
|
|
|
363
364
|
},
|
|
364
365
|
async releaseLease(key) {
|
|
365
366
|
assertNotShutdown();
|
|
366
|
-
const etcdKey =
|
|
367
|
+
const etcdKey = keys.lease(key);
|
|
367
368
|
const entry = leaseManager.remove(etcdKey);
|
|
368
369
|
if (entry !== void 0) {
|
|
369
370
|
await entry.lease.revoke().catch(() => {
|
|
@@ -373,7 +374,7 @@ async function createEtcdCoordinator(config) {
|
|
|
373
374
|
},
|
|
374
375
|
async get(key) {
|
|
375
376
|
assertNotShutdown();
|
|
376
|
-
const etcdKey =
|
|
377
|
+
const etcdKey = keys.generic(key);
|
|
377
378
|
const buf = await client.get(etcdKey).buffer();
|
|
378
379
|
if (buf === null) {
|
|
379
380
|
return null;
|
|
@@ -382,7 +383,7 @@ async function createEtcdCoordinator(config) {
|
|
|
382
383
|
},
|
|
383
384
|
async compareAndSet(key, expected, value) {
|
|
384
385
|
assertNotShutdown();
|
|
385
|
-
const etcdKey =
|
|
386
|
+
const etcdKey = keys.generic(key);
|
|
386
387
|
const newValue = Buffer.from(value);
|
|
387
388
|
if (expected === null) {
|
|
388
389
|
const txn2 = await client.if(etcdKey, "Create", "==", 0).then(client.put(etcdKey).value(newValue)).commit();
|
|
@@ -394,7 +395,7 @@ async function createEtcdCoordinator(config) {
|
|
|
394
395
|
},
|
|
395
396
|
async getSchema(indexName) {
|
|
396
397
|
assertNotShutdown();
|
|
397
|
-
const key =
|
|
398
|
+
const key = keys.schema(indexName);
|
|
398
399
|
const buf = await client.get(key).buffer();
|
|
399
400
|
if (buf === null) {
|
|
400
401
|
return null;
|
|
@@ -403,12 +404,12 @@ async function createEtcdCoordinator(config) {
|
|
|
403
404
|
},
|
|
404
405
|
async putSchema(indexName, schema) {
|
|
405
406
|
assertNotShutdown();
|
|
406
|
-
const key =
|
|
407
|
+
const key = keys.schema(indexName);
|
|
407
408
|
await client.put(key).value(serializeSchema(schema)).exec();
|
|
408
409
|
},
|
|
409
410
|
async watchSchemas(handler) {
|
|
410
411
|
assertNotShutdown();
|
|
411
|
-
const prefix =
|
|
412
|
+
const prefix = keys.schemaPrefix();
|
|
412
413
|
const watcher = await client.watch().prefix(prefix).create();
|
|
413
414
|
addWatcher(watcher);
|
|
414
415
|
watcher.on("put", (kv) => {
|
|
@@ -429,7 +430,7 @@ async function createEtcdCoordinator(config) {
|
|
|
429
430
|
},
|
|
430
431
|
async getLeaseHolder(key) {
|
|
431
432
|
assertNotShutdown();
|
|
432
|
-
const etcdKey =
|
|
433
|
+
const etcdKey = keys.lease(key);
|
|
433
434
|
const value = await client.get(etcdKey).string();
|
|
434
435
|
return value ?? null;
|
|
435
436
|
},
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// src/languages/support/spellings.ts
|
|
2
|
+
function withNormalisedSpellings(words, normalize) {
|
|
3
|
+
const listed = [...words];
|
|
4
|
+
const expanded = new Set(listed);
|
|
5
|
+
for (const word of listed) expanded.add(normalize(word));
|
|
6
|
+
return expanded;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export { withNormalisedSpellings };
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
// src/languages/support/marks.ts
|
|
2
|
+
function isAscii(token) {
|
|
3
|
+
for (let i = 0; i < token.length; i++) {
|
|
4
|
+
if (token.charCodeAt(i) > 127) return false;
|
|
5
|
+
}
|
|
6
|
+
return true;
|
|
7
|
+
}
|
|
8
|
+
function removeMarks(token, marks) {
|
|
9
|
+
if (isAscii(token)) return token;
|
|
10
|
+
const decomposed = token.normalize("NFD");
|
|
11
|
+
const stripped = decomposed.replace(marks, "");
|
|
12
|
+
return stripped === decomposed ? token : stripped.normalize("NFC");
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export { removeMarks };
|