@delali/narsil 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +777 -130
- package/dist/{chunk-GQEHCZG7.mjs → chunk-CU6BVZEU.mjs} +643 -447
- package/dist/{chunk-FS7XLIUS.mjs → chunk-Q4OVNESD.mjs} +52 -4
- package/dist/{config-cQLlYR3U.d.ts → config-DX--2Owg.d.ts} +3 -1
- package/dist/{direct-executor-FX56DYD3.mjs → direct-executor-6FOKJKLD.mjs} +1 -1
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +4 -4
- package/dist/distribution.mjs +2 -2
- package/dist/embeddings/openai.mjs +5 -0
- package/dist/index.browser.mjs +476 -244
- package/dist/index.d.ts +3 -3
- package/dist/index.mjs +2 -2
- package/dist/{narsil-ChTdkwPu.d.ts → narsil-B3jKyXCr.d.ts} +6 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +2 -2
- package/dist/{schema-BVaw7nqz.d.ts → schema-C3VhNvfw.d.ts} +3 -1
- package/dist/server.d.ts +3 -3
- package/dist/server.mjs +11 -12
- package/dist/{types-Bt7c4hDF.d.ts → types-CTVVm4GG.d.ts} +1 -1
- package/dist/workers/entry.mjs +1 -1
- package/package.json +11 -2
|
@@ -738,6 +738,38 @@ function extractVectorFieldsFromSchema(schema) {
|
|
|
738
738
|
return result;
|
|
739
739
|
}
|
|
740
740
|
|
|
741
|
+
// src/schema/validator/vector-promotion.ts
|
|
742
|
+
function fail(message, details) {
|
|
743
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, message, details);
|
|
744
|
+
}
|
|
745
|
+
function isPositiveInteger(value) {
|
|
746
|
+
return Number.isInteger(value) && value >= 1;
|
|
747
|
+
}
|
|
748
|
+
function validateVectorPromotion(config) {
|
|
749
|
+
if (config === void 0) return;
|
|
750
|
+
if (config.threshold !== void 0 && !isPositiveInteger(config.threshold)) {
|
|
751
|
+
fail("vectorPromotion.threshold must be a positive integer", { threshold: config.threshold });
|
|
752
|
+
}
|
|
753
|
+
const quantization = config.quantization;
|
|
754
|
+
if (quantization !== void 0 && quantization !== "sq8" && quantization !== "none") {
|
|
755
|
+
fail("vectorPromotion.quantization must be 'sq8' or 'none'", { quantization });
|
|
756
|
+
}
|
|
757
|
+
const hnsw = config.hnswConfig;
|
|
758
|
+
if (hnsw === void 0) return;
|
|
759
|
+
if (hnsw.m !== void 0 && !isPositiveInteger(hnsw.m)) {
|
|
760
|
+
fail("vectorPromotion.hnswConfig.m must be a positive integer", { m: hnsw.m });
|
|
761
|
+
}
|
|
762
|
+
if (hnsw.efConstruction !== void 0 && !isPositiveInteger(hnsw.efConstruction)) {
|
|
763
|
+
fail("vectorPromotion.hnswConfig.efConstruction must be a positive integer", {
|
|
764
|
+
efConstruction: hnsw.efConstruction
|
|
765
|
+
});
|
|
766
|
+
}
|
|
767
|
+
const metric = hnsw.metric;
|
|
768
|
+
if (metric !== void 0 && metric !== "cosine" && metric !== "dotProduct" && metric !== "euclidean") {
|
|
769
|
+
fail("vectorPromotion.hnswConfig.metric must be 'cosine', 'dotProduct', or 'euclidean'", { metric });
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
741
773
|
// src/search/fulltext.ts
|
|
742
774
|
function fulltextSearch(partition, params, language, schema, options) {
|
|
743
775
|
if (!params.term || params.term.trim().length === 0) {
|
|
@@ -4392,6 +4424,24 @@ function deserializeScalarQuantizer(data, dimensions, ordinalSource) {
|
|
|
4392
4424
|
return quantizer;
|
|
4393
4425
|
}
|
|
4394
4426
|
|
|
4427
|
+
// src/workers/node-worker.ts
|
|
4428
|
+
async function spawnNodeWorker(entryPoint) {
|
|
4429
|
+
try {
|
|
4430
|
+
const workerThreads = await import('worker_threads');
|
|
4431
|
+
return new workerThreads.Worker(entryPoint);
|
|
4432
|
+
} catch {
|
|
4433
|
+
return null;
|
|
4434
|
+
}
|
|
4435
|
+
}
|
|
4436
|
+
async function isNodeMainThread() {
|
|
4437
|
+
try {
|
|
4438
|
+
const workerThreads = await import('worker_threads');
|
|
4439
|
+
return workerThreads.isMainThread;
|
|
4440
|
+
} catch {
|
|
4441
|
+
return true;
|
|
4442
|
+
}
|
|
4443
|
+
}
|
|
4444
|
+
|
|
4395
4445
|
// src/runtime/detect.ts
|
|
4396
4446
|
function detectRuntimeName() {
|
|
4397
4447
|
if ("Bun" in globalThis) {
|
|
@@ -4441,9 +4491,7 @@ async function spawnWorker() {
|
|
|
4441
4491
|
const entryPoint = resolveWorkerEntryPoint();
|
|
4442
4492
|
if (runtime.supportsWorkerThreads) {
|
|
4443
4493
|
try {
|
|
4444
|
-
|
|
4445
|
-
const instance = new workerThreads.Worker(new URL(entryPoint));
|
|
4446
|
-
return instance;
|
|
4494
|
+
return await spawnNodeWorker(new URL(entryPoint));
|
|
4447
4495
|
} catch {
|
|
4448
4496
|
return null;
|
|
4449
4497
|
}
|
|
@@ -5293,4 +5341,4 @@ function createDirectExecutor() {
|
|
|
5293
5341
|
};
|
|
5294
5342
|
}
|
|
5295
5343
|
|
|
5296
|
-
export { MAX_NESTING_DEPTH, clearNormalizationCache, configureNormalizationCache, createDirectExecutor, createPartitionIndex, createPartitionManager, createPartitionRouter, createVectorIndex, deserializePayloadV2, detectRuntime, extractVectorFieldsFromSchema, fanOutQuery, flattenSchema, fnv1a, getLanguage, getNestedValue, getNormalizationCacheSize, kWayMerge, mergePartitionStats, registerLanguage, resetNormalizationCache, tokenize, validateRequiredFields, validateSchema };
|
|
5344
|
+
export { MAX_NESTING_DEPTH, clearNormalizationCache, configureNormalizationCache, createDirectExecutor, createPartitionIndex, createPartitionManager, createPartitionRouter, createVectorIndex, deserializePayloadV2, detectRuntime, extractVectorFieldsFromSchema, fanOutQuery, flattenSchema, fnv1a, getLanguage, getNestedValue, getNormalizationCacheSize, isNodeMainThread, kWayMerge, mergePartitionStats, registerLanguage, resetNormalizationCache, spawnNodeWorker, tokenize, validateRequiredFields, validateSchema, validateVectorPromotion };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { b as PersistenceAdapter, I as InvalidationAdapter, E as EmbeddingAdapter } from './adapters-Bo2JQg8b.js';
|
|
2
|
-
import { A as AnyDocument, b as ScoringMode, S as SchemaDefinition, I as IndexConfig } from './schema-
|
|
2
|
+
import { A as AnyDocument, b as ScoringMode, S as SchemaDefinition, I as IndexConfig } from './schema-C3VhNvfw.js';
|
|
3
3
|
|
|
4
4
|
declare const ErrorCodes: {
|
|
5
5
|
readonly SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE";
|
|
@@ -445,6 +445,8 @@ interface NarsilConfig {
|
|
|
445
445
|
workers?: WorkerConfig;
|
|
446
446
|
flush?: FlushConfig;
|
|
447
447
|
embedding?: EmbeddingAdapter;
|
|
448
|
+
/** Named adapters; names persist in index metadata so recovery can rebind. */
|
|
449
|
+
embeddingAdapters?: Record<string, EmbeddingAdapter>;
|
|
448
450
|
durability?: DurabilityConfig;
|
|
449
451
|
}
|
|
450
452
|
interface DurabilityConfig {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { C as ClusterCoordinator } from '../../types-
|
|
2
|
-
import '../../schema-
|
|
1
|
+
import { C as ClusterCoordinator } from '../../types-CTVVm4GG.js';
|
|
2
|
+
import '../../schema-C3VhNvfw.js';
|
|
3
3
|
import '../../adapters-Bo2JQg8b.js';
|
|
4
4
|
|
|
5
5
|
interface EtcdCoordinatorConfig {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { C as ClusterCoordinator } from '../../types-
|
|
2
|
-
import '../../schema-
|
|
1
|
+
import { C as ClusterCoordinator } from '../../types-CTVVm4GG.js';
|
|
2
|
+
import '../../schema-C3VhNvfw.js';
|
|
3
3
|
import '../../adapters-Bo2JQg8b.js';
|
|
4
4
|
|
|
5
5
|
declare function createInMemoryCoordinator(): ClusterCoordinator;
|
package/dist/distribution.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { C as ClusterCoordinator, N as NodeRegistration, A as AllocationTable, a as NodeRole, b as NodeCapacity } from './types-
|
|
2
|
-
export { c as AllocationConstraints, d as AllocationEvent, e as NodeEvent, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-
|
|
3
|
-
import { B as BatchResult, Q as QueryParams, u as QueryResult, N as NarsilConfig } from './config-
|
|
4
|
-
import { I as IndexConfig, A as AnyDocument } from './schema-
|
|
1
|
+
import { C as ClusterCoordinator, N as NodeRegistration, A as AllocationTable, a as NodeRole, b as NodeCapacity } from './types-CTVVm4GG.js';
|
|
2
|
+
export { c as AllocationConstraints, d as AllocationEvent, e as NodeEvent, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-CTVVm4GG.js';
|
|
3
|
+
import { B as BatchResult, Q as QueryParams, u as QueryResult, N as NarsilConfig } from './config-DX--2Owg.js';
|
|
4
|
+
import { I as IndexConfig, A as AnyDocument } from './schema-C3VhNvfw.js';
|
|
5
5
|
export { createInMemoryCoordinator } from './distribution/coordinator/in-memory.js';
|
|
6
6
|
import './adapters-Bo2JQg8b.js';
|
|
7
7
|
|
package/dist/distribution.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
export { createInMemoryCoordinator } from './chunk-RQLXC2YS.mjs';
|
|
2
2
|
import './chunk-WISGW24C.mjs';
|
|
3
|
-
import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-
|
|
4
|
-
import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-
|
|
3
|
+
import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-CU6BVZEU.mjs';
|
|
4
|
+
import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-Q4OVNESD.mjs';
|
|
5
5
|
import './chunk-S7RJJMHI.mjs';
|
|
6
6
|
import { crc32 } from './chunk-KMMJMDUA.mjs';
|
|
7
7
|
import './chunk-TW6BXTOL.mjs';
|
|
@@ -129,6 +129,11 @@ async function executeWithRetry(url, requestBody, apiKey, maxRetries, signal) {
|
|
|
129
129
|
try {
|
|
130
130
|
parsed = await response.json();
|
|
131
131
|
} catch {
|
|
132
|
+
lastError = new Error("invalid JSON in a 2xx response body");
|
|
133
|
+
if (attempt < maxRetries) {
|
|
134
|
+
await sleep(computeBackoffMs(attempt, null), signal);
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
132
137
|
throw new NarsilError(ErrorCodes.EMBEDDING_FAILED, "OpenAI returned invalid JSON in response body");
|
|
133
138
|
}
|
|
134
139
|
if (parsed === null || typeof parsed !== "object") {
|