@delali/narsil 0.3.1-next.79 → 0.3.1-next.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-XSZTF4ZQ.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-DFW3U5UW.mjs → chunk-4CWHWOBI.mjs} +24 -195
- package/dist/{chunk-R6WBLBTC.mjs → chunk-4D6OH4EF.mjs} +10 -46
- package/dist/{chunk-RTR44V2A.mjs → chunk-4VBG6PEH.mjs} +586 -4
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-PJWG547Z.mjs → chunk-GNYXCLJS.mjs} +5 -5
- package/dist/{chunk-55WEJKWT.mjs → chunk-IH3GKNDQ.mjs} +197 -93
- package/dist/{chunk-AMGDG4ZJ.mjs → chunk-IN5CH4P2.mjs} +3 -3
- package/dist/{chunk-PXDQFYD5.mjs → chunk-KJ7LB3TR.mjs} +24 -140
- package/dist/chunk-LS5IRM23.mjs +140 -0
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
- package/dist/{chunk-4344I4MR.mjs → chunk-SREPGC6O.mjs} +336 -13
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-P2BEMMOD.mjs → chunk-XYDP7CCJ.mjs} +151 -204
- package/dist/{chunk-Z3BXCZXS.mjs → chunk-XZF4LZ3X.mjs} +5 -3
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-ZCKPRCYI.mjs +30 -0
- package/dist/{client-ZXcbSViH.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-DHL2KsfD.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-DTe5EbWW.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-JVXDR5Y5.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -2
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +2 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +759 -65
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-BjPBzWkA.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +138 -539
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +17 -15
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/{memory-DW3OJ0im.d.ts → memory-BPKNx9Tl.d.ts} +1288 -1287
- package/dist/partial-words.mjs +6 -5
- package/dist/pattern.mjs +5 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +21 -13
- package/dist/{tasks-BTbwBqar.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-XBDVXKUE.mjs → thread-33M5AOEI.mjs} +13 -10
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/workers/entry.mjs +5 -5
- package/package.json +13 -7
- package/dist/chunk-IOZPBNSG.mjs +0 -55
- package/dist/chunk-WFVOBZS4.mjs +0 -485
- package/dist/direct-executor-WB2HKU4N.mjs +0 -17
|
@@ -1,19 +1,24 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
|
-
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter,
|
|
5
|
-
import { requirePatternSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, kWayMerge, mergeFacets,
|
|
6
|
-
import { detectRuntime, getLanguage } from './chunk-
|
|
7
|
-
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-
|
|
8
|
-
import { releaseLocksHeldBy } from './chunk-
|
|
9
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
10
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES
|
|
11
|
-
import { getNestedValue, parseVectorDimension } from './chunk-
|
|
12
|
-
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema,
|
|
3
|
+
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, deadBitsWhere, NO_LIVE_VECTOR_AT_POSITION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-4CWHWOBI.mjs';
|
|
5
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
|
+
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
8
|
+
import { releaseLocksHeldBy } from './chunk-IN5CH4P2.mjs';
|
|
9
|
+
import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
|
|
10
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
|
|
11
|
+
import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
|
|
12
|
+
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
|
|
13
13
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
14
|
-
import {
|
|
14
|
+
import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
15
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
15
16
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
|
|
16
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
17
|
+
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
18
|
+
import { openSync, close, readFileSync } from 'fs';
|
|
19
|
+
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
20
|
+
import { endianness } from 'os';
|
|
21
|
+
import { dirname } from 'path';
|
|
17
22
|
import { encode, decode } from '@msgpack/msgpack';
|
|
18
23
|
import * as zlib from 'zlib';
|
|
19
24
|
|
|
@@ -727,15 +732,48 @@ async function getPath() {
|
|
|
727
732
|
}
|
|
728
733
|
return cachedPath;
|
|
729
734
|
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
var
|
|
733
|
-
var
|
|
734
|
-
var
|
|
735
|
+
var AUXV_CLOCK_TICKS_TYPE = 17n;
|
|
736
|
+
var AUXV_END_TYPE = 0n;
|
|
737
|
+
var THIRTY_TWO_BIT_ARCHITECTURES = /* @__PURE__ */ new Set(["arm", "ia32", "mips", "mipsel", "ppc"]);
|
|
738
|
+
var FIRST_STAT_FIELD_AFTER_COMMAND = 3;
|
|
739
|
+
var STAT_STATE_FIELD = 3;
|
|
740
|
+
var STAT_START_TIME_FIELD = 22;
|
|
741
|
+
var EXITED_STATES = /* @__PURE__ */ new Set(["Z", "X", "x"]);
|
|
742
|
+
var UNKNOWN = { kind: "unknown" };
|
|
743
|
+
var clockTicksPerSecond = null;
|
|
735
744
|
function errnoOf(err) {
|
|
736
745
|
return err?.code;
|
|
737
746
|
}
|
|
738
|
-
function
|
|
747
|
+
function readWord(bytes, offset, wordBytes, littleEndian) {
|
|
748
|
+
if (wordBytes === 8) return littleEndian ? bytes.readBigUInt64LE(offset) : bytes.readBigUInt64BE(offset);
|
|
749
|
+
return BigInt(littleEndian ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset));
|
|
750
|
+
}
|
|
751
|
+
async function readClockTicksPerSecond() {
|
|
752
|
+
const auxv = await readFile("/proc/self/auxv");
|
|
753
|
+
const wordBytes = THIRTY_TWO_BIT_ARCHITECTURES.has(process.arch) ? 4 : 8;
|
|
754
|
+
const littleEndian = endianness() === "LE";
|
|
755
|
+
for (let offset = 0; offset + 2 * wordBytes <= auxv.length; offset += 2 * wordBytes) {
|
|
756
|
+
const type = readWord(auxv, offset, wordBytes, littleEndian);
|
|
757
|
+
if (type === AUXV_END_TYPE) return null;
|
|
758
|
+
if (type === AUXV_CLOCK_TICKS_TYPE) {
|
|
759
|
+
const ticks = Number(readWord(auxv, offset + wordBytes, wordBytes, littleEndian));
|
|
760
|
+
return ticks > 0 ? ticks : null;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
return null;
|
|
764
|
+
}
|
|
765
|
+
function clockTicks() {
|
|
766
|
+
clockTicksPerSecond ??= readClockTicksPerSecond().catch(() => null);
|
|
767
|
+
return clockTicksPerSecond;
|
|
768
|
+
}
|
|
769
|
+
function threadGroupOf(status) {
|
|
770
|
+
const leader = /^Tgid:\s*(\d+)\s*$/m.exec(status)?.[1];
|
|
771
|
+
return leader === void 0 ? null : Number(leader);
|
|
772
|
+
}
|
|
773
|
+
function statField(fieldsAfterCommand, field) {
|
|
774
|
+
return fieldsAfterCommand[field - FIRST_STAT_FIELD_AFTER_COMMAND];
|
|
775
|
+
}
|
|
776
|
+
function signalReaches(pid) {
|
|
739
777
|
try {
|
|
740
778
|
process.kill(pid, 0);
|
|
741
779
|
return true;
|
|
@@ -743,6 +781,101 @@ function processIsRunning(pid) {
|
|
|
743
781
|
return errnoOf(err) === "EPERM";
|
|
744
782
|
}
|
|
745
783
|
}
|
|
784
|
+
async function identifyProcess(pid) {
|
|
785
|
+
try {
|
|
786
|
+
const [status, stat, ticksPerSecond] = await Promise.all([
|
|
787
|
+
readFile(`/proc/${pid}/status`, "utf8"),
|
|
788
|
+
readFile(`/proc/${pid}/stat`, "utf8"),
|
|
789
|
+
clockTicks()
|
|
790
|
+
]);
|
|
791
|
+
const leader = threadGroupOf(status);
|
|
792
|
+
if (leader === null) return UNKNOWN;
|
|
793
|
+
if (leader !== pid) return { kind: "thread" };
|
|
794
|
+
const commandEnd = stat.lastIndexOf(")");
|
|
795
|
+
if (commandEnd < 0) return UNKNOWN;
|
|
796
|
+
const fieldsAfterCommand = stat.slice(commandEnd + 1).trim().split(/\s+/);
|
|
797
|
+
const state = statField(fieldsAfterCommand, STAT_STATE_FIELD);
|
|
798
|
+
if (state !== void 0 && EXITED_STATES.has(state)) return { kind: "exited" };
|
|
799
|
+
const startTicks = Number(statField(fieldsAfterCommand, STAT_START_TIME_FIELD));
|
|
800
|
+
const uptimeSeconds = Number.parseFloat(readFileSync("/proc/uptime", "utf8"));
|
|
801
|
+
const nowMs = Date.now();
|
|
802
|
+
if (ticksPerSecond === null || !Number.isFinite(startTicks) || !Number.isFinite(uptimeSeconds)) return UNKNOWN;
|
|
803
|
+
return { kind: "process", startMs: Math.round(nowMs - (uptimeSeconds - startTicks / ticksPerSecond) * 1e3) };
|
|
804
|
+
} catch {
|
|
805
|
+
return UNKNOWN;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
async function descriptorTargets(pid) {
|
|
809
|
+
const descriptorDirectory = `/proc/${pid}/fd`;
|
|
810
|
+
let descriptors;
|
|
811
|
+
try {
|
|
812
|
+
descriptors = await readdir(descriptorDirectory);
|
|
813
|
+
} catch {
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
816
|
+
const targets = [];
|
|
817
|
+
for (const descriptor of descriptors) {
|
|
818
|
+
try {
|
|
819
|
+
targets.push(await readlink(`${descriptorDirectory}/${descriptor}`));
|
|
820
|
+
} catch (err) {
|
|
821
|
+
if (errnoOf(err) !== "ENOENT") return null;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return targets;
|
|
825
|
+
}
|
|
826
|
+
async function holdsFileInDirectoryOf(pid, lockPath) {
|
|
827
|
+
const targets = await descriptorTargets(pid);
|
|
828
|
+
if (targets === null) return false;
|
|
829
|
+
const directory = await realpath(dirname(lockPath)).catch(() => null);
|
|
830
|
+
if (directory === null) return false;
|
|
831
|
+
const inside = directory.endsWith("/") ? directory : `${directory}/`;
|
|
832
|
+
return targets.some((target) => target === directory || target.startsWith(inside));
|
|
833
|
+
}
|
|
834
|
+
async function holderIsLive(holder, self, lockPath) {
|
|
835
|
+
if (holder.pid === self.pid) {
|
|
836
|
+
if (holder.processStartMs === null || self.processStartMs === null) return false;
|
|
837
|
+
if (Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS) return true;
|
|
838
|
+
return holdsFileInDirectoryOf(holder.pid, lockPath);
|
|
839
|
+
}
|
|
840
|
+
if (!signalReaches(holder.pid)) return false;
|
|
841
|
+
const found = await identifyProcess(holder.pid);
|
|
842
|
+
if (found.kind === "thread" || found.kind === "exited") return false;
|
|
843
|
+
if (found.kind === "unknown") return signalReaches(holder.pid);
|
|
844
|
+
if (holder.processStartMs === null) return true;
|
|
845
|
+
if (found.startMs - holder.processStartMs <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS) return true;
|
|
846
|
+
return holdsFileInDirectoryOf(holder.pid, lockPath);
|
|
847
|
+
}
|
|
848
|
+
function holdDirectoryOpen(directory) {
|
|
849
|
+
if (process.platform !== "linux") return () => void 0;
|
|
850
|
+
const descriptor = openSync(directory, "r");
|
|
851
|
+
return () => close(descriptor, () => void 0);
|
|
852
|
+
}
|
|
853
|
+
async function removeStaleLock(lockPath, asidePath, isStale) {
|
|
854
|
+
try {
|
|
855
|
+
await rename(lockPath, asidePath);
|
|
856
|
+
} catch (err) {
|
|
857
|
+
if (errnoOf(err) === "ENOENT") return;
|
|
858
|
+
throw err;
|
|
859
|
+
}
|
|
860
|
+
try {
|
|
861
|
+
const moved = await readFile(asidePath, "utf8");
|
|
862
|
+
if (!isStale(moved)) {
|
|
863
|
+
await writeFile(lockPath, moved, { flag: "wx" }).catch((err) => {
|
|
864
|
+
if (errnoOf(err) !== "EEXIST") throw err;
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
} finally {
|
|
868
|
+
await unlink(asidePath).catch(() => void 0);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// src/persistence/durability/directory-lock.ts
|
|
873
|
+
var LOCK_FILE_NAME = ".narsil.lock";
|
|
874
|
+
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
875
|
+
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
876
|
+
function errnoOf2(err) {
|
|
877
|
+
return err?.code;
|
|
878
|
+
}
|
|
746
879
|
function integerOrNull(text) {
|
|
747
880
|
const value = Number.parseInt(text ?? "", 10);
|
|
748
881
|
return Number.isSafeInteger(value) ? value : null;
|
|
@@ -770,7 +903,7 @@ async function readHolder(fs, lockPath) {
|
|
|
770
903
|
try {
|
|
771
904
|
return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
|
|
772
905
|
} catch (err) {
|
|
773
|
-
if (
|
|
906
|
+
if (errnoOf2(err) === "ENOENT") return "absent";
|
|
774
907
|
throw err;
|
|
775
908
|
}
|
|
776
909
|
}
|
|
@@ -780,10 +913,9 @@ async function settledHolder(fs, lockPath) {
|
|
|
780
913
|
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
781
914
|
return readHolder(fs, lockPath);
|
|
782
915
|
}
|
|
783
|
-
function
|
|
784
|
-
if (
|
|
785
|
-
|
|
786
|
-
return Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS;
|
|
916
|
+
function sameReading(first, second) {
|
|
917
|
+
if (typeof first !== "object" || typeof second !== "object") return first === second;
|
|
918
|
+
return formatHolder(first) === formatHolder(second);
|
|
787
919
|
}
|
|
788
920
|
function heldElsewhere(root, holder, lockPath) {
|
|
789
921
|
return new NarsilError(
|
|
@@ -801,7 +933,7 @@ async function createLockFileExclusively(fs, lockPath, content) {
|
|
|
801
933
|
await fs.writeFile(lockPath, content, { flag: "wx" });
|
|
802
934
|
return true;
|
|
803
935
|
} catch (err) {
|
|
804
|
-
if (
|
|
936
|
+
if (errnoOf2(err) === "EEXIST") return false;
|
|
805
937
|
throw err;
|
|
806
938
|
}
|
|
807
939
|
}
|
|
@@ -811,7 +943,7 @@ async function createLockFile(fs, lockPath, draftPath, content) {
|
|
|
811
943
|
await fs.link(draftPath, lockPath);
|
|
812
944
|
return true;
|
|
813
945
|
} catch (err) {
|
|
814
|
-
const code =
|
|
946
|
+
const code = errnoOf2(err);
|
|
815
947
|
if (code === "EEXIST") return false;
|
|
816
948
|
if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
|
|
817
949
|
return createLockFileExclusively(fs, lockPath, content);
|
|
@@ -827,12 +959,14 @@ async function acquire(fs, root, lockPath, self) {
|
|
|
827
959
|
for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
|
|
828
960
|
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
829
961
|
const holder = await settledHolder(fs, lockPath);
|
|
830
|
-
if (typeof holder === "object" && holderIsLive(holder, self)) {
|
|
962
|
+
if (typeof holder === "object" && await holderIsLive(holder, self, lockPath)) {
|
|
831
963
|
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
832
964
|
}
|
|
833
|
-
await
|
|
834
|
-
|
|
835
|
-
|
|
965
|
+
await removeStaleLock(
|
|
966
|
+
lockPath,
|
|
967
|
+
`${draftPath}.aside`,
|
|
968
|
+
(text) => sameReading(holder, parseHolder(text) ?? "unreadable")
|
|
969
|
+
);
|
|
836
970
|
}
|
|
837
971
|
throw heldElsewhere(root, "another process", lockPath);
|
|
838
972
|
}
|
|
@@ -850,7 +984,15 @@ async function lockDirectory(root) {
|
|
|
850
984
|
const resolvedRoot = pathMod.resolve(root);
|
|
851
985
|
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
852
986
|
const self = await describeSelf();
|
|
987
|
+
async function holds() {
|
|
988
|
+
const holder = await readHolder(fs, lockPath);
|
|
989
|
+
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
990
|
+
}
|
|
991
|
+
async function removeOwnLockFile() {
|
|
992
|
+
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
993
|
+
}
|
|
853
994
|
let directoryKey = null;
|
|
995
|
+
let closeDirectory = () => void 0;
|
|
854
996
|
try {
|
|
855
997
|
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
856
998
|
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
@@ -859,8 +1001,10 @@ async function lockDirectory(root) {
|
|
|
859
1001
|
lockedDirectories.add(key);
|
|
860
1002
|
directoryKey = key;
|
|
861
1003
|
await acquire(fs, root, lockPath, self);
|
|
1004
|
+
closeDirectory = holdDirectoryOpen(resolvedRoot);
|
|
862
1005
|
} catch (err) {
|
|
863
1006
|
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
1007
|
+
await removeOwnLockFile();
|
|
864
1008
|
if (err instanceof NarsilError) throw err;
|
|
865
1009
|
throw new NarsilError(
|
|
866
1010
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
@@ -870,17 +1014,14 @@ async function lockDirectory(root) {
|
|
|
870
1014
|
}
|
|
871
1015
|
const heldKey = directoryKey;
|
|
872
1016
|
let released = false;
|
|
873
|
-
async function holds() {
|
|
874
|
-
const holder = await readHolder(fs, lockPath);
|
|
875
|
-
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
876
|
-
}
|
|
877
1017
|
return {
|
|
878
1018
|
holds,
|
|
879
1019
|
async release() {
|
|
880
1020
|
if (released) return;
|
|
881
1021
|
released = true;
|
|
882
1022
|
lockedDirectories.delete(heldKey);
|
|
883
|
-
|
|
1023
|
+
await removeOwnLockFile();
|
|
1024
|
+
closeDirectory();
|
|
884
1025
|
}
|
|
885
1026
|
};
|
|
886
1027
|
}
|
|
@@ -2988,6 +3129,13 @@ function createInsyncConfirmMessage(payload, sourceId, requestId) {
|
|
|
2988
3129
|
function isRecord2(value) {
|
|
2989
3130
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2990
3131
|
}
|
|
3132
|
+
function isPlainObject(value) {
|
|
3133
|
+
if (!isRecord2(value)) {
|
|
3134
|
+
return false;
|
|
3135
|
+
}
|
|
3136
|
+
const prototype = Object.getPrototypeOf(value);
|
|
3137
|
+
return prototype === Object.prototype || prototype === null;
|
|
3138
|
+
}
|
|
2991
3139
|
function validateEntryPayload(decoded) {
|
|
2992
3140
|
if (!isRecord2(decoded) || !isRecord2(decoded.entry)) {
|
|
2993
3141
|
throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
|
|
@@ -3109,6 +3257,9 @@ function validateForwardBatchResultPayload(decoded) {
|
|
|
3109
3257
|
if (result.errorMessage !== null && typeof result.errorMessage !== "string") {
|
|
3110
3258
|
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorMessage" must be a string or null`);
|
|
3111
3259
|
}
|
|
3260
|
+
if (result.errorDetails !== void 0 && result.errorDetails !== null && !isPlainObject(result.errorDetails)) {
|
|
3261
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorDetails" must be an object or absent`);
|
|
3262
|
+
}
|
|
3112
3263
|
}
|
|
3113
3264
|
return decoded;
|
|
3114
3265
|
}
|
|
@@ -7365,7 +7516,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
7365
7516
|
if (indexState === void 0) {
|
|
7366
7517
|
return;
|
|
7367
7518
|
}
|
|
7368
|
-
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), (
|
|
7519
|
+
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), (close2) => close2);
|
|
7369
7520
|
indexes.delete(indexName);
|
|
7370
7521
|
metadataWrites.delete(indexName);
|
|
7371
7522
|
},
|
|
@@ -10324,65 +10475,18 @@ function stopIdleSweep(state) {
|
|
|
10324
10475
|
}
|
|
10325
10476
|
|
|
10326
10477
|
// src/core/pattern-index/matcher-budget.ts
|
|
10327
|
-
var POSITIONS_PER_WORD = 32;
|
|
10328
|
-
function positionWords(positions) {
|
|
10329
|
-
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
10330
|
-
}
|
|
10331
|
-
function smallestMatcherBytes(positions) {
|
|
10332
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
10333
|
-
}
|
|
10334
10478
|
function widestPatternSplit(footprintBytes) {
|
|
10335
10479
|
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / Math.max(1, footprintBytes)));
|
|
10336
10480
|
}
|
|
10337
10481
|
|
|
10338
10482
|
// src/core/pattern-index/query-positions.ts
|
|
10339
|
-
function isRecord4(value) {
|
|
10340
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
10341
|
-
}
|
|
10342
|
-
function literalFootprint(text, caseFold) {
|
|
10343
|
-
if (typeof text !== "string") return null;
|
|
10344
|
-
return smallestMatcherBytes(caseFold ? foldedCodePointCount(text) : codePointCount(text));
|
|
10345
|
-
}
|
|
10346
|
-
function larger(left, right) {
|
|
10347
|
-
if (left === null) return right;
|
|
10348
|
-
if (right === null) return left;
|
|
10349
|
-
return Math.max(left, right);
|
|
10350
|
-
}
|
|
10351
|
-
function largestTestOf(filter, module) {
|
|
10352
|
-
const caseFold = filter.caseFold === true;
|
|
10353
|
-
let largest = null;
|
|
10354
|
-
for (const operator of PATTERN_TEST_OPERATORS) {
|
|
10355
|
-
const operand = filter[operator];
|
|
10356
|
-
if (operand === void 0) continue;
|
|
10357
|
-
const texts = Array.isArray(operand) ? operand : [operand];
|
|
10358
|
-
for (const text of texts) largest = larger(largest, literalFootprint(text, caseFold));
|
|
10359
|
-
}
|
|
10360
|
-
for (const operator of PATTERN_EXPRESSION_OPERATORS) {
|
|
10361
|
-
const pattern = filter[operator];
|
|
10362
|
-
if (typeof pattern !== "string") continue;
|
|
10363
|
-
largest = larger(largest, module.expressionFootprint(operator, pattern, caseFold, PATTERN_HOST));
|
|
10364
|
-
}
|
|
10365
|
-
return largest;
|
|
10366
|
-
}
|
|
10367
|
-
function largestIn(expression, patternFields, module) {
|
|
10368
|
-
let largest = null;
|
|
10369
|
-
if (isRecord4(expression.fields)) {
|
|
10370
|
-
for (const [fieldPath, filter] of Object.entries(expression.fields)) {
|
|
10371
|
-
if (patternFields.has(fieldPath) && isRecord4(filter)) largest = larger(largest, largestTestOf(filter, module));
|
|
10372
|
-
}
|
|
10373
|
-
}
|
|
10374
|
-
const nested = [...expression.and ?? [], ...expression.or ?? []];
|
|
10375
|
-
if (expression.not !== void 0) nested.push(expression.not);
|
|
10376
|
-
for (const child of nested) largest = larger(largest, largestIn(child, patternFields, module));
|
|
10377
|
-
return largest;
|
|
10378
|
-
}
|
|
10379
10483
|
function largestPatternFootprint(expression, schema) {
|
|
10380
10484
|
if (expression === void 0) return null;
|
|
10381
10485
|
const module = availablePatternSearch();
|
|
10382
10486
|
if (module === null) return null;
|
|
10383
10487
|
const patternFields = /* @__PURE__ */ new Set();
|
|
10384
10488
|
for (const { path, pattern } of schemaFieldsOf(schema)) if (pattern) patternFields.add(path);
|
|
10385
|
-
return patternFields.size === 0 ? null :
|
|
10489
|
+
return patternFields.size === 0 ? null : module.largestFootprint(expression, patternFields, PATTERN_HOST);
|
|
10386
10490
|
}
|
|
10387
10491
|
|
|
10388
10492
|
// src/engine/orchestration/search.ts
|
|
@@ -13096,7 +13200,7 @@ function refuse(path, reason) {
|
|
|
13096
13200
|
field: path
|
|
13097
13201
|
});
|
|
13098
13202
|
}
|
|
13099
|
-
function
|
|
13203
|
+
function isRecord4(value) {
|
|
13100
13204
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13101
13205
|
}
|
|
13102
13206
|
function startsWithTagPrefix(key) {
|
|
@@ -13163,7 +13267,7 @@ function untagTag(tag, value, path, depth) {
|
|
|
13163
13267
|
case NUMBER_TAG:
|
|
13164
13268
|
return numberOf(value, path);
|
|
13165
13269
|
case OBJECT_TAG:
|
|
13166
|
-
if (!
|
|
13270
|
+
if (!isRecord4(value)) refuse(path, `holds an "${OBJECT_TAG}" tag whose value is not an object`);
|
|
13167
13271
|
return untagMap(value, path, depth);
|
|
13168
13272
|
default:
|
|
13169
13273
|
return refuse(path, `holds an object whose only key, "${tag}", names no tag`);
|
|
@@ -13386,7 +13490,7 @@ function documentExportStream(header2, checkpoint) {
|
|
|
13386
13490
|
function invalidHeader(message) {
|
|
13387
13491
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `The document export header ${message}`);
|
|
13388
13492
|
}
|
|
13389
|
-
function
|
|
13493
|
+
function isRecord5(value) {
|
|
13390
13494
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
13391
13495
|
}
|
|
13392
13496
|
function configOfHeader(line) {
|
|
@@ -13402,7 +13506,7 @@ function configOfHeader(line) {
|
|
|
13402
13506
|
} catch {
|
|
13403
13507
|
invalidHeader("is not JSON");
|
|
13404
13508
|
}
|
|
13405
|
-
if (!
|
|
13509
|
+
if (!isRecord5(raw) || raw.format !== DOCUMENT_EXPORT_FORMAT || raw.version !== DOCUMENT_EXPORT_VERSION) {
|
|
13406
13510
|
invalidHeader(`must name format "${DOCUMENT_EXPORT_FORMAT}" and version ${DOCUMENT_EXPORT_VERSION}`);
|
|
13407
13511
|
}
|
|
13408
13512
|
const configuration = requireConfigurationShape(raw.index, "document export header");
|
|
@@ -13495,7 +13599,7 @@ async function takeLine(state, entry) {
|
|
|
13495
13599
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line is not JSON");
|
|
13496
13600
|
return;
|
|
13497
13601
|
}
|
|
13498
|
-
if (!
|
|
13602
|
+
if (!isRecord5(document) || typeof document.id !== "string" || document.id.length === 0) {
|
|
13499
13603
|
refuse3(state, entry.line, ErrorCodes.DOC_VALIDATION_FAILED, "The line holds no document with a string id");
|
|
13500
13604
|
return;
|
|
13501
13605
|
}
|
|
@@ -14114,17 +14218,17 @@ function rejectConfiguration(message) {
|
|
|
14114
14218
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
|
|
14115
14219
|
}
|
|
14116
14220
|
function renamed(manifest, sourceName, indexName) {
|
|
14117
|
-
const
|
|
14221
|
+
const rename2 = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
|
|
14118
14222
|
return {
|
|
14119
14223
|
...manifest,
|
|
14120
14224
|
partitions: manifest.partitions.map((partition) => ({
|
|
14121
14225
|
...partition,
|
|
14122
|
-
segments: partition.segments.map((segment) => ({ ...segment, key:
|
|
14226
|
+
segments: partition.segments.map((segment) => ({ ...segment, key: rename2(segment.key) }))
|
|
14123
14227
|
})),
|
|
14124
14228
|
vectors: manifest.vectors.map((field) => ({
|
|
14125
14229
|
...field,
|
|
14126
|
-
files: field.files.map((file) => ({ ...file, key:
|
|
14127
|
-
graphParts: field.graphParts.map((part) => ({ ...part, key:
|
|
14230
|
+
files: field.files.map((file) => ({ ...file, key: rename2(file.key) })),
|
|
14231
|
+
graphParts: field.graphParts.map((part) => ({ ...part, key: rename2(part.key) }))
|
|
14128
14232
|
}))
|
|
14129
14233
|
};
|
|
14130
14234
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
|
|
2
|
-
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-
|
|
2
|
+
import { nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback } from './chunk-B3UOQJD3.mjs';
|
|
3
3
|
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-KEHK2O6P.mjs';
|
|
4
|
-
import { compareCodePoints } from './chunk-
|
|
5
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
6
6
|
import { readSync, closeSync, openSync } from 'fs';
|
|
7
7
|
|
|
8
8
|
// src/vector/constants.ts
|
|
@@ -1,145 +1,9 @@
|
|
|
1
1
|
import { EARLIER_RELEASE_MIN_M, MIN_M, MAX_M } from './chunk-KEHK2O6P.mjs';
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { DEFAULT_PATTERN_VALUE_LIMIT, isPlainObject, isWordIndexedBase, MAX_TEXT_FIELDS, MAX_VECTOR_DIMENSION, parseFieldType, MAX_DOCUMENT_BYTES, MAX_NESTING_DEPTH, PROTOTYPE_POLLUTION_KEYS, FIELD_NAME_PATTERN, RESERVED_ROOT_FIELDS, MAX_DOCUMENT_NESTING_DEPTH, VECTOR_PATTERN } from './chunk-LS5IRM23.mjs';
|
|
3
|
+
import { compareCodePoints, holdsUnpairedSurrogate, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
4
5
|
import { encode } from '@msgpack/msgpack';
|
|
5
6
|
|
|
6
|
-
// src/schema/constants.ts
|
|
7
|
-
var MAX_NESTING_DEPTH = 4;
|
|
8
|
-
var MAX_DOCUMENT_NESTING_DEPTH = 32;
|
|
9
|
-
var DEFAULT_PATTERN_VALUE_LIMIT = 8192;
|
|
10
|
-
var MIN_PATTERN_VALUE_LIMIT = 1;
|
|
11
|
-
var MAX_PATTERN_VALUE_LIMIT = 65536;
|
|
12
|
-
var MAX_VECTOR_DIMENSION = 4096;
|
|
13
|
-
var MAX_STORED_VECTOR_DIMENSION = 65535;
|
|
14
|
-
var MAX_TEXT_FIELDS = 65536;
|
|
15
|
-
var MAX_DOCUMENT_BYTES = 16777216;
|
|
16
|
-
var PARSED_FIELD_TYPE_CACHE_ENTRIES = 1024;
|
|
17
|
-
|
|
18
|
-
// src/schema/validator/shared.ts
|
|
19
|
-
var VECTOR_PATTERN = /^vector\[(\d+)]$/;
|
|
20
|
-
function isGeopointOrVectorType(fieldType) {
|
|
21
|
-
return fieldType === "geopoint" || fieldType !== void 0 && VECTOR_PATTERN.test(fieldType);
|
|
22
|
-
}
|
|
23
|
-
var FIELD_NAME_PATTERN = /^[A-Za-z0-9_]+$/;
|
|
24
|
-
var RESERVED_ROOT_FIELDS = /* @__PURE__ */ new Set(["id"]);
|
|
25
|
-
var PROTOTYPE_POLLUTION_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
26
|
-
function isPlainObject(value) {
|
|
27
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// src/schema/validator/field-type.ts
|
|
31
|
-
var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];
|
|
32
|
-
var OPTIONS_BY_BASE = {
|
|
33
|
-
string: FIELD_OPTION_ORDER,
|
|
34
|
-
"string[]": FIELD_OPTION_ORDER,
|
|
35
|
-
verbatim: ["sortable"],
|
|
36
|
-
"verbatim[]": ["sortable"],
|
|
37
|
-
number: [],
|
|
38
|
-
"number[]": [],
|
|
39
|
-
boolean: [],
|
|
40
|
-
"boolean[]": [],
|
|
41
|
-
enum: [],
|
|
42
|
-
"enum[]": [],
|
|
43
|
-
geopoint: [],
|
|
44
|
-
vector: []
|
|
45
|
-
};
|
|
46
|
-
var TYPE_SEPARATOR = ":";
|
|
47
|
-
function isNamedBase(name) {
|
|
48
|
-
return name !== "vector" && Object.hasOwn(OPTIONS_BY_BASE, name);
|
|
49
|
-
}
|
|
50
|
-
function describeAllowedOptions(baseName, allowed) {
|
|
51
|
-
if (allowed.length === 0) return `the "${baseName}" type has no options`;
|
|
52
|
-
const quoted = allowed.map((option) => `"${option}"`);
|
|
53
|
-
if (quoted.length === 1) return `the only option of the "${baseName}" type is ${quoted[0]}`;
|
|
54
|
-
return `the options of the "${baseName}" type are ${quoted.slice(0, -1).join(", ")}, and ${quoted[quoted.length - 1]}`;
|
|
55
|
-
}
|
|
56
|
-
function invalidType(path, type, message) {
|
|
57
|
-
throw new NarsilError(ErrorCodes.SCHEMA_INVALID_TYPE, message, { field: path, type });
|
|
58
|
-
}
|
|
59
|
-
function parseBase(baseName, path, type) {
|
|
60
|
-
if (isNamedBase(baseName)) return { base: baseName, dimension: 0 };
|
|
61
|
-
const vectorMatch = VECTOR_PATTERN.exec(baseName);
|
|
62
|
-
if (vectorMatch === null) {
|
|
63
|
-
invalidType(path, type, `Field "${path}" has an unsupported type: "${type}"`);
|
|
64
|
-
}
|
|
65
|
-
const dimension = Number.parseInt(vectorMatch[1], 10);
|
|
66
|
-
if (dimension <= 0 || dimension > MAX_STORED_VECTOR_DIMENSION) {
|
|
67
|
-
throw new NarsilError(
|
|
68
|
-
ErrorCodes.SCHEMA_INVALID_VECTOR_DIMENSION,
|
|
69
|
-
`Vector field "${path}" has ${dimension} dimensions, but a vector field holds from 1 to ${MAX_STORED_VECTOR_DIMENSION}`,
|
|
70
|
-
{ field: path, dimension, maximum: MAX_STORED_VECTOR_DIMENSION }
|
|
71
|
-
);
|
|
72
|
-
}
|
|
73
|
-
return { base: "vector", dimension };
|
|
74
|
-
}
|
|
75
|
-
function parseFieldType(type, path) {
|
|
76
|
-
const separatorIndex = type.indexOf(TYPE_SEPARATOR);
|
|
77
|
-
const baseName = separatorIndex === -1 ? type : type.slice(0, separatorIndex);
|
|
78
|
-
const { base, dimension } = parseBase(baseName, path, type);
|
|
79
|
-
const allowed = OPTIONS_BY_BASE[base];
|
|
80
|
-
const options = /* @__PURE__ */ new Set();
|
|
81
|
-
if (separatorIndex !== -1) {
|
|
82
|
-
for (const option of type.slice(separatorIndex + 1).split(TYPE_SEPARATOR)) {
|
|
83
|
-
if (!allowed.includes(option)) {
|
|
84
|
-
invalidType(
|
|
85
|
-
path,
|
|
86
|
-
type,
|
|
87
|
-
`Field "${path}" has type "${type}" with the option "${option}", but ${describeAllowedOptions(baseName, allowed)}`
|
|
88
|
-
);
|
|
89
|
-
}
|
|
90
|
-
if (options.has(option)) {
|
|
91
|
-
invalidType(path, type, `Field "${path}" has the "${option}" option twice in type "${type}"`);
|
|
92
|
-
}
|
|
93
|
-
options.add(option);
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
const ordered = FIELD_OPTION_ORDER.filter((option) => options.has(option));
|
|
97
|
-
const verbatim = base === "verbatim" || base === "verbatim[]";
|
|
98
|
-
return {
|
|
99
|
-
type: [baseName, ...ordered].join(TYPE_SEPARATOR),
|
|
100
|
-
base,
|
|
101
|
-
sortable: options.has("sortable"),
|
|
102
|
-
pattern: verbatim || options.has("pattern"),
|
|
103
|
-
partial: options.has("partial"),
|
|
104
|
-
dimension
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
var parsedTypesByName = /* @__PURE__ */ new Map();
|
|
108
|
-
function parsedTypeOf(type) {
|
|
109
|
-
if (type === void 0) return void 0;
|
|
110
|
-
const known = parsedTypesByName.get(type);
|
|
111
|
-
if (known !== void 0) return known;
|
|
112
|
-
try {
|
|
113
|
-
const parsed = Object.freeze(parseFieldType(type, ""));
|
|
114
|
-
if (parsedTypesByName.size < PARSED_FIELD_TYPE_CACHE_ENTRIES) parsedTypesByName.set(type, parsed);
|
|
115
|
-
return parsed;
|
|
116
|
-
} catch {
|
|
117
|
-
return void 0;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
function withSortableOption(parsed) {
|
|
121
|
-
if (parsed.sortable) return parsed.type;
|
|
122
|
-
const separatorIndex = parsed.type.indexOf(TYPE_SEPARATOR);
|
|
123
|
-
const baseName = separatorIndex === -1 ? parsed.type : parsed.type.slice(0, separatorIndex);
|
|
124
|
-
return parseFieldType(`${baseName}${TYPE_SEPARATOR}sortable${parsed.type.slice(baseName.length)}`, "").type;
|
|
125
|
-
}
|
|
126
|
-
function isListBase(base) {
|
|
127
|
-
return base.endsWith("[]");
|
|
128
|
-
}
|
|
129
|
-
function isWordIndexedBase(base) {
|
|
130
|
-
return base === "string" || base === "string[]";
|
|
131
|
-
}
|
|
132
|
-
function isSingleStringFieldType(fieldType) {
|
|
133
|
-
return parsedTypeOf(fieldType)?.base === "string";
|
|
134
|
-
}
|
|
135
|
-
function isWordIndexedFieldType(fieldType) {
|
|
136
|
-
const parsed = parsedTypeOf(fieldType);
|
|
137
|
-
return parsed !== void 0 && isWordIndexedBase(parsed.base);
|
|
138
|
-
}
|
|
139
|
-
function isStringValuedBase(base) {
|
|
140
|
-
return base === "string" || base === "string[]" || base === "verbatim" || base === "verbatim[]";
|
|
141
|
-
}
|
|
142
|
-
|
|
143
7
|
// src/schema/validator/schema.ts
|
|
144
8
|
function validateSchemaFields(schema, depth, prefix) {
|
|
145
9
|
if (depth > MAX_NESTING_DEPTH) {
|
|
@@ -801,4 +665,24 @@ function validateVectorPromotion(config) {
|
|
|
801
665
|
}
|
|
802
666
|
}
|
|
803
667
|
|
|
804
|
-
|
|
668
|
+
// src/search/constants.ts
|
|
669
|
+
var RESULT_WINDOW = 1e4;
|
|
670
|
+
var DEFAULT_PAGE_SIZE = 10;
|
|
671
|
+
var MAX_CURSOR_LENGTH = 40960;
|
|
672
|
+
var MAX_SORT_FIELDS = 8;
|
|
673
|
+
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
674
|
+
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
675
|
+
var PREFIX_MAX_EXPANSIONS = 50;
|
|
676
|
+
var PARTIAL_WORDS_ADDED_PER_QUERY = 50;
|
|
677
|
+
var PARTIAL_WORD_MIN_CODE_POINTS = 3;
|
|
678
|
+
var DEFAULT_FUZZY_PREFIX_LENGTH = 2;
|
|
679
|
+
var MAX_TOLERANCE = 10;
|
|
680
|
+
var MAX_PREFIX_LENGTH = 1024;
|
|
681
|
+
var DEFAULT_MAX_PER_GROUP = 1;
|
|
682
|
+
var DEFAULT_RANK_CONSTANT = 60;
|
|
683
|
+
var MIN_RANK_CONSTANT = 1;
|
|
684
|
+
var DEFAULT_LINEAR_ALPHA = 0.5;
|
|
685
|
+
var MIN_LINEAR_ALPHA = 0;
|
|
686
|
+
var MAX_LINEAR_ALPHA = 1;
|
|
687
|
+
|
|
688
|
+
export { ANTIMERIDIAN_SPAN_DEGREES, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_LINEAR_ALPHA, DEFAULT_MAX_PER_GROUP, DEFAULT_PAGE_SIZE, DEFAULT_RANK_CONSTANT, DEG_TO_RAD, EARTH_RADIUS_METERS, FULL_TURN_DEGREES, LOCATION_CODE_BITS_PER_AXIS, MAX_COVERING_CELLS, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_LATITUDE_DEGREES, MAX_LINEAR_ALPHA, MAX_LONGITUDE_DEGREES, MAX_PREFIX_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_TOLERANCE, MIN_LINEAR_ALPHA, MIN_POLYGON_POINTS, MIN_RANK_CONSTANT, PARTIAL_WORDS_ADDED_PER_QUERY, PARTIAL_WORD_MIN_CODE_POINTS, PREFIX_MAX_EXPANSIONS, RADIUS_BOX_MARGIN_RADIANS, RADIUS_BOX_MARGIN_RATIO, RESULT_WINDOW, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, encodeDocument, extractVectorFieldsFromSchema, flattenSchema, isLatitude, isLongitude, isQuantizationMode, isStorageMode, messagePackUpperBoundBytes, partialFieldsOf, raiseEarlierReleaseGraphSetting, requireCreatableSchema, requireDocumentWithinSizeLimit, requireValidFieldTypes, schemaFieldsOf, validateDocument, validateDocumentStrict, validateRequiredFields, validateSchema, validateVectorPromotion, validateVectorStorage };
|