@delali/narsil 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/README.md +44 -2
  2. package/dist/adapters/filesystem.d.ts +1 -1
  3. package/dist/adapters/filesystem.mjs +46 -2
  4. package/dist/adapters/indexeddb.d.ts +1 -1
  5. package/dist/adapters/indexeddb.mjs +1 -1
  6. package/dist/adapters/memory.d.ts +1 -1
  7. package/dist/{adapters-BbRkoiu9.d.ts → adapters-Bo2JQg8b.d.ts} +2 -1
  8. package/dist/chunk-ALB35PRQ.mjs +98 -0
  9. package/dist/{chunk-KHL4RPC3.mjs → chunk-FS7XLIUS.mjs} +2536 -2284
  10. package/dist/chunk-GQEHCZG7.mjs +6825 -0
  11. package/dist/chunk-KMMJMDUA.mjs +44 -0
  12. package/dist/chunk-RQLXC2YS.mjs +269 -0
  13. package/dist/chunk-S7RJJMHI.mjs +1086 -0
  14. package/dist/{chunk-I24SGLFS.mjs → chunk-TW6BXTOL.mjs} +5 -1
  15. package/dist/chunk-WISGW24C.mjs +455 -0
  16. package/dist/config-cQLlYR3U.d.ts +470 -0
  17. package/dist/direct-executor-FX56DYD3.mjs +4 -0
  18. package/dist/distribution/coordinator/etcd.d.ts +16 -0
  19. package/dist/distribution/coordinator/etcd.mjs +2 -0
  20. package/dist/distribution/coordinator/in-memory.d.ts +7 -0
  21. package/dist/distribution/coordinator/in-memory.mjs +1 -0
  22. package/dist/distribution.d.ts +122 -0
  23. package/dist/distribution.mjs +7410 -0
  24. package/dist/embeddings/openai.d.ts +1 -1
  25. package/dist/embeddings/openai.mjs +1 -1
  26. package/dist/index.browser.mjs +11469 -7481
  27. package/dist/index.d.ts +6 -450
  28. package/dist/index.mjs +7 -3236
  29. package/dist/invalidation/broadcast-channel.d.ts +1 -1
  30. package/dist/invalidation/filesystem.d.ts +1 -1
  31. package/dist/invalidation/noop.d.ts +1 -1
  32. package/dist/{language-ClUGLuoC.d.ts → language-ONQ3iVDf.d.ts} +1 -0
  33. package/dist/languages/arabic.d.ts +1 -1
  34. package/dist/languages/armenian.d.ts +1 -1
  35. package/dist/languages/bulgarian.d.ts +1 -1
  36. package/dist/languages/chinese.d.ts +1 -1
  37. package/dist/languages/dagbani.d.ts +1 -1
  38. package/dist/languages/danish.d.ts +1 -1
  39. package/dist/languages/dutch.d.ts +1 -1
  40. package/dist/languages/english.d.ts +1 -1
  41. package/dist/languages/english.mjs +1 -1
  42. package/dist/languages/ewe.d.ts +1 -1
  43. package/dist/languages/finnish.d.ts +1 -1
  44. package/dist/languages/french.d.ts +1 -1
  45. package/dist/languages/ga.d.ts +1 -1
  46. package/dist/languages/german.d.ts +1 -1
  47. package/dist/languages/greek.d.ts +1 -1
  48. package/dist/languages/hausa.d.ts +1 -1
  49. package/dist/languages/hindi.d.ts +1 -1
  50. package/dist/languages/hungarian.d.ts +1 -1
  51. package/dist/languages/igbo.d.ts +1 -1
  52. package/dist/languages/indonesian.d.ts +1 -1
  53. package/dist/languages/irish.d.ts +1 -1
  54. package/dist/languages/italian.d.ts +1 -1
  55. package/dist/languages/japanese.d.ts +1 -1
  56. package/dist/languages/nepali.d.ts +1 -1
  57. package/dist/languages/norwegian.d.ts +1 -1
  58. package/dist/languages/portuguese.d.ts +1 -1
  59. package/dist/languages/romanian.d.ts +1 -1
  60. package/dist/languages/russian.d.ts +1 -1
  61. package/dist/languages/sanskrit.d.ts +1 -1
  62. package/dist/languages/serbian.d.ts +1 -1
  63. package/dist/languages/slovenian.d.ts +1 -1
  64. package/dist/languages/spanish.d.ts +1 -1
  65. package/dist/languages/swahili.d.ts +1 -1
  66. package/dist/languages/swedish.d.ts +1 -1
  67. package/dist/languages/tamil.d.ts +1 -1
  68. package/dist/languages/turkish.d.ts +1 -1
  69. package/dist/languages/twi.d.ts +1 -1
  70. package/dist/languages/ukrainian.d.ts +1 -1
  71. package/dist/languages/yoruba.d.ts +1 -1
  72. package/dist/languages/zulu.d.ts +1 -1
  73. package/dist/narsil-ChTdkwPu.d.ts +68 -0
  74. package/dist/persistence/durability/checkpoint-worker.d.ts +41 -0
  75. package/dist/persistence/durability/checkpoint-worker.mjs +59 -0
  76. package/dist/schema-BVaw7nqz.d.ts +58 -0
  77. package/dist/serialization/crc32-worker.d.ts +20 -0
  78. package/dist/serialization/crc32-worker.mjs +61 -0
  79. package/dist/server.d.ts +218 -0
  80. package/dist/server.mjs +1185 -0
  81. package/dist/types-Bt7c4hDF.d.ts +73 -0
  82. package/dist/vector/hnsw-build-worker.mjs +1 -1
  83. package/dist/workers/entry.mjs +1 -1
  84. package/package.json +37 -2
  85. package/dist/chunk-R2C2EQGO.mjs +0 -53
  86. package/dist/chunk-VVORJOXR.mjs +0 -759
  87. package/dist/direct-executor-LBJLGMJ2.mjs +0 -4
@@ -0,0 +1,44 @@
1
+ // src/serialization/crc32.ts
2
+ var IEEE_POLYNOMIAL = 3988292384;
3
+ var cachedTable = null;
4
+ function buildTable() {
5
+ const table = new Uint32Array(256);
6
+ for (let i = 0; i < 256; i++) {
7
+ let crc = i;
8
+ for (let bit = 0; bit < 8; bit++) {
9
+ crc = crc & 1 ? crc >>> 1 ^ IEEE_POLYNOMIAL : crc >>> 1;
10
+ }
11
+ table[i] = crc;
12
+ }
13
+ return table;
14
+ }
15
+ function getTable() {
16
+ if (cachedTable === null) {
17
+ cachedTable = buildTable();
18
+ }
19
+ return cachedTable;
20
+ }
21
+ function crc32(data) {
22
+ const table = getTable();
23
+ let crc = 4294967295;
24
+ for (let i = 0; i < data.length; i++) {
25
+ crc = crc >>> 8 ^ table[(crc ^ data[i]) & 255];
26
+ }
27
+ return (crc ^ 4294967295) >>> 0;
28
+ }
29
+ function crc32Init() {
30
+ return 4294967295;
31
+ }
32
+ function crc32Update(state, data) {
33
+ const table = getTable();
34
+ let crc = state;
35
+ for (let i = 0; i < data.length; i++) {
36
+ crc = crc >>> 8 ^ table[(crc ^ data[i]) & 255];
37
+ }
38
+ return crc;
39
+ }
40
+ function crc32Final(state) {
41
+ return (state ^ 4294967295) >>> 0;
42
+ }
43
+
44
+ export { crc32, crc32Final, crc32Init, crc32Update };
@@ -0,0 +1,269 @@
1
+ // src/distribution/coordinator/in-memory.ts
2
+ var NODE_HEARTBEAT_PREFIX = "_narsil/node/";
3
+ function bytesEqual(a, b) {
4
+ if (a.byteLength !== b.byteLength) {
5
+ return false;
6
+ }
7
+ for (let i = 0; i < a.byteLength; i++) {
8
+ if (a[i] !== b[i]) {
9
+ return false;
10
+ }
11
+ }
12
+ return true;
13
+ }
14
+ function createInMemoryCoordinator() {
15
+ const nodes = /* @__PURE__ */ new Map();
16
+ const allocations = /* @__PURE__ */ new Map();
17
+ const partitionStates = /* @__PURE__ */ new Map();
18
+ const leases = /* @__PURE__ */ new Map();
19
+ const schemas = /* @__PURE__ */ new Map();
20
+ const kvStore = /* @__PURE__ */ new Map();
21
+ const nodeWatchers = /* @__PURE__ */ new Set();
22
+ const allocationWatchers = /* @__PURE__ */ new Set();
23
+ const schemaWatchers = /* @__PURE__ */ new Set();
24
+ let isShutdown = false;
25
+ function assertNotShutdown() {
26
+ if (isShutdown) {
27
+ throw new Error("Coordinator has been shut down");
28
+ }
29
+ }
30
+ function emitNodeEvent(event) {
31
+ for (const handler of nodeWatchers) {
32
+ try {
33
+ handler(event);
34
+ } catch (_) {
35
+ }
36
+ }
37
+ }
38
+ function emitAllocationEvent(event) {
39
+ for (const handler of allocationWatchers) {
40
+ try {
41
+ handler(event);
42
+ } catch (_) {
43
+ }
44
+ }
45
+ }
46
+ function emitSchemaEvent(event) {
47
+ for (const handler of schemaWatchers) {
48
+ try {
49
+ handler(event);
50
+ } catch (_) {
51
+ }
52
+ }
53
+ }
54
+ function clearLeaseEntry(key) {
55
+ const entry = leases.get(key);
56
+ if (entry !== void 0) {
57
+ clearTimeout(entry.timer);
58
+ leases.delete(key);
59
+ }
60
+ }
61
+ function createLeaseTimer(key, ttlMs) {
62
+ return setTimeout(() => {
63
+ leases.delete(key);
64
+ if (key.startsWith(NODE_HEARTBEAT_PREFIX)) {
65
+ const expiredNodeId = key.slice(NODE_HEARTBEAT_PREFIX.length);
66
+ nodes.delete(expiredNodeId);
67
+ emitNodeEvent({ type: "node_left", nodeId: expiredNodeId, registration: null });
68
+ }
69
+ }, ttlMs);
70
+ }
71
+ function isLeaseValid(key) {
72
+ const entry = leases.get(key);
73
+ if (entry === void 0) {
74
+ return false;
75
+ }
76
+ return entry.expiresAt > Date.now();
77
+ }
78
+ const coordinator = {
79
+ async registerNode(registration) {
80
+ assertNotShutdown();
81
+ nodes.set(registration.nodeId, registration);
82
+ const heartbeatKey = `${NODE_HEARTBEAT_PREFIX}${registration.nodeId}`;
83
+ const defaultTtlMs = 3e4;
84
+ clearLeaseEntry(heartbeatKey);
85
+ const timer = createLeaseTimer(heartbeatKey, defaultTtlMs);
86
+ leases.set(heartbeatKey, {
87
+ nodeId: registration.nodeId,
88
+ expiresAt: Date.now() + defaultTtlMs,
89
+ timer
90
+ });
91
+ emitNodeEvent({ type: "node_joined", nodeId: registration.nodeId, registration });
92
+ },
93
+ async deregisterNode(nodeId) {
94
+ assertNotShutdown();
95
+ const existed = nodes.delete(nodeId);
96
+ const heartbeatKey = `${NODE_HEARTBEAT_PREFIX}${nodeId}`;
97
+ clearLeaseEntry(heartbeatKey);
98
+ if (existed) {
99
+ emitNodeEvent({ type: "node_left", nodeId, registration: null });
100
+ }
101
+ },
102
+ async listNodes() {
103
+ assertNotShutdown();
104
+ return Array.from(nodes.values());
105
+ },
106
+ async watchNodes(handler) {
107
+ assertNotShutdown();
108
+ nodeWatchers.add(handler);
109
+ return () => {
110
+ nodeWatchers.delete(handler);
111
+ };
112
+ },
113
+ async getAllocation(indexName) {
114
+ assertNotShutdown();
115
+ return allocations.get(indexName) ?? null;
116
+ },
117
+ async putAllocation(indexName, table, expectedVersion) {
118
+ assertNotShutdown();
119
+ if (expectedVersion !== void 0) {
120
+ const current = allocations.get(indexName);
121
+ if (expectedVersion === null) {
122
+ if (current !== void 0) {
123
+ return false;
124
+ }
125
+ } else {
126
+ if (current === void 0 || current.version !== expectedVersion) {
127
+ return false;
128
+ }
129
+ }
130
+ }
131
+ allocations.set(indexName, table);
132
+ emitAllocationEvent({ indexName, table });
133
+ return true;
134
+ },
135
+ async watchAllocation(handler) {
136
+ assertNotShutdown();
137
+ allocationWatchers.add(handler);
138
+ return () => {
139
+ allocationWatchers.delete(handler);
140
+ };
141
+ },
142
+ async getPartitionState(indexName, partitionId) {
143
+ assertNotShutdown();
144
+ const indexStates = partitionStates.get(indexName);
145
+ if (indexStates === void 0) {
146
+ return "UNASSIGNED";
147
+ }
148
+ return indexStates.get(partitionId) ?? "UNASSIGNED";
149
+ },
150
+ async putPartitionState(indexName, partitionId, state) {
151
+ assertNotShutdown();
152
+ let indexStates = partitionStates.get(indexName);
153
+ if (indexStates === void 0) {
154
+ indexStates = /* @__PURE__ */ new Map();
155
+ partitionStates.set(indexName, indexStates);
156
+ }
157
+ indexStates.set(partitionId, state);
158
+ },
159
+ async acquireLease(key, nodeId, ttlMs) {
160
+ assertNotShutdown();
161
+ if (isLeaseValid(key)) {
162
+ const entry = leases.get(key);
163
+ if (entry !== void 0 && entry.nodeId !== nodeId) {
164
+ return false;
165
+ }
166
+ if (entry !== void 0 && entry.nodeId === nodeId) {
167
+ clearTimeout(entry.timer);
168
+ const timer2 = createLeaseTimer(key, ttlMs);
169
+ entry.expiresAt = Date.now() + ttlMs;
170
+ entry.timer = timer2;
171
+ return true;
172
+ }
173
+ }
174
+ clearLeaseEntry(key);
175
+ const timer = createLeaseTimer(key, ttlMs);
176
+ leases.set(key, { nodeId, expiresAt: Date.now() + ttlMs, timer });
177
+ return true;
178
+ },
179
+ async renewLease(key, nodeId, ttlMs) {
180
+ assertNotShutdown();
181
+ const entry = leases.get(key);
182
+ if (entry === void 0 || entry.nodeId !== nodeId) {
183
+ return false;
184
+ }
185
+ if (!isLeaseValid(key)) {
186
+ leases.delete(key);
187
+ return false;
188
+ }
189
+ clearTimeout(entry.timer);
190
+ entry.timer = createLeaseTimer(key, ttlMs);
191
+ entry.expiresAt = Date.now() + ttlMs;
192
+ return true;
193
+ },
194
+ async releaseLease(key) {
195
+ assertNotShutdown();
196
+ clearLeaseEntry(key);
197
+ },
198
+ async get(key) {
199
+ assertNotShutdown();
200
+ const value = kvStore.get(key);
201
+ return value !== void 0 ? new Uint8Array(value) : null;
202
+ },
203
+ async compareAndSet(key, expected, value) {
204
+ assertNotShutdown();
205
+ const current = kvStore.get(key);
206
+ if (expected === null) {
207
+ if (current !== void 0) {
208
+ return false;
209
+ }
210
+ kvStore.set(key, new Uint8Array(value));
211
+ return true;
212
+ }
213
+ if (current === void 0) {
214
+ return false;
215
+ }
216
+ if (!bytesEqual(current, expected)) {
217
+ return false;
218
+ }
219
+ kvStore.set(key, new Uint8Array(value));
220
+ return true;
221
+ },
222
+ async getSchema(indexName) {
223
+ assertNotShutdown();
224
+ return schemas.get(indexName) ?? null;
225
+ },
226
+ async putSchema(indexName, schema) {
227
+ assertNotShutdown();
228
+ schemas.set(indexName, schema);
229
+ emitSchemaEvent({ type: "schema_created", indexName, schema });
230
+ },
231
+ async watchSchemas(handler) {
232
+ assertNotShutdown();
233
+ schemaWatchers.add(handler);
234
+ return () => {
235
+ schemaWatchers.delete(handler);
236
+ };
237
+ },
238
+ async getLeaseHolder(key) {
239
+ assertNotShutdown();
240
+ if (!isLeaseValid(key)) {
241
+ clearLeaseEntry(key);
242
+ return null;
243
+ }
244
+ const entry = leases.get(key);
245
+ return entry?.nodeId ?? null;
246
+ },
247
+ async shutdown() {
248
+ if (isShutdown) {
249
+ return;
250
+ }
251
+ isShutdown = true;
252
+ for (const entry of leases.values()) {
253
+ clearTimeout(entry.timer);
254
+ }
255
+ leases.clear();
256
+ nodes.clear();
257
+ allocations.clear();
258
+ partitionStates.clear();
259
+ schemas.clear();
260
+ kvStore.clear();
261
+ nodeWatchers.clear();
262
+ allocationWatchers.clear();
263
+ schemaWatchers.clear();
264
+ }
265
+ };
266
+ return coordinator;
267
+ }
268
+
269
+ export { createInMemoryCoordinator };