@delali/narsil 0.3.1-next.80 → 0.3.1-next.83
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-CcTDWh_P.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-HJASKX4F.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-4QDQEJDV.mjs → chunk-4D6OH4EF.mjs} +6 -6
- package/dist/{chunk-P2PYEZZE.mjs → chunk-4VBG6PEH.mjs} +3 -3
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-NSXQCE4K.mjs → chunk-7JZZ3HNV.mjs} +208 -48
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-K4FZ55Q2.mjs → chunk-CUP2E4OH.mjs} +175 -99
- package/dist/{chunk-MEOOYKWL.mjs → chunk-GNYXCLJS.mjs} +4 -4
- package/dist/{chunk-OSF3SGJV.mjs → chunk-KJ7LB3TR.mjs} +2 -2
- package/dist/{chunk-DQABW5UQ.mjs → chunk-LS5IRM23.mjs} +1 -1
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-23HJ5QXQ.mjs → chunk-RZXNWQSM.mjs} +183 -94
- package/dist/{chunk-R2XCJIO7.mjs → chunk-SREPGC6O.mjs} +2 -2
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/{chunk-YGLSR7BC.mjs → chunk-XYDP7CCJ.mjs} +5 -5
- package/dist/{chunk-QPYV22XH.mjs → chunk-XZF4LZ3X.mjs} +4 -3
- package/dist/{chunk-GG2FBGVZ.mjs → chunk-ZCKPRCYI.mjs} +1 -1
- package/dist/{client-CsybN673.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-CFonR_gY.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-BCNQtszf.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-6HASZ36Y.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +756 -64
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-B0oIF1_r.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +426 -249
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +14 -14
- package/dist/{memory-B6SBOKlS.d.ts → memory-BPKNx9Tl.d.ts} +1 -0
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +18 -13
- package/dist/{tasks-CYAQe6qd.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-P6QJ2Q5O.mjs → thread-33M5AOEI.mjs} +10 -10
- package/dist/vector/search-worker.d.ts +2 -0
- package/dist/vector/search-worker.mjs +8 -7
- package/dist/workers/entry.mjs +5 -5
- package/package.json +11 -7
- package/dist/direct-executor-VNMQ6NIH.mjs +0 -19
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { E as EmbeddingAdapter } from './adapters-79OIa4xm.js';
|
|
2
|
-
import { C as ClusterNamespace } from './config-
|
|
3
|
-
import { h as TaskStore } from './tasks-
|
|
2
|
+
import { C as ClusterNamespace } from './config-BuIkeM5C.js';
|
|
3
|
+
import { h as TaskStore } from './tasks-DGyz586z.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
6
|
* Context handed to {@link OnRequestHook} for every inbound request, captured
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-
|
|
2
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { requireDocumentWithinSizeLimit, validateRequiredFields } from './chunk-KJ7LB3TR.mjs';
|
|
2
|
+
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
3
3
|
|
|
4
4
|
// src/narsil/internals.ts
|
|
5
5
|
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-
|
|
2
|
-
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, fnv1aBytes, validatedFilterOf, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, partialWordsToHighlight, partialWordHighlighter, undeclaredFieldOnStrictIndex } from './chunk-XYDP7CCJ.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-ZCKPRCYI.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
4
|
-
import { tokenize } from './chunk-
|
|
5
|
-
import { partialFieldsOf, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-
|
|
4
|
+
import { tokenize } from './chunk-4VBG6PEH.mjs';
|
|
5
|
+
import { partialFieldsOf, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-KJ7LB3TR.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-
|
|
7
|
+
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isWordIndexedFieldType, isGeopointOrVectorType } from './chunk-LS5IRM23.mjs';
|
|
8
8
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
9
9
|
import { holdsUnpairedSurrogate, compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
10
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
10
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
11
11
|
|
|
12
12
|
// src/search/cursor-binding.ts
|
|
13
13
|
var ABSENT_TAG = 0;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { flattenSchema, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-
|
|
1
|
+
import { flattenSchema, PARTIAL_WORD_MIN_CODE_POINTS, PARTIAL_WORDS_ADDED_PER_QUERY, DEFAULT_FUZZY_PREFIX_LENGTH } from './chunk-KJ7LB3TR.mjs';
|
|
2
2
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
3
|
-
import { isWordIndexedBase } from './chunk-
|
|
3
|
+
import { isWordIndexedBase } from './chunk-LS5IRM23.mjs';
|
|
4
4
|
import { compareCodePoints, codePointCount } from './chunk-YTJCGKM4.mjs';
|
|
5
|
-
import { createNarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { createNarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
6
6
|
|
|
7
7
|
// src/core/tokenizer/constants.ts
|
|
8
8
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { performance } from 'perf_hooks';
|
|
2
|
-
import {
|
|
2
|
+
import { setFlagsFromString, getHeapStatistics } from 'v8';
|
|
3
3
|
import { runInNewContext } from 'vm';
|
|
4
4
|
|
|
5
5
|
// src/runtime/idle-heap-cleanup.ts
|
|
@@ -93,4 +93,4 @@ function startIdleHeapCleanup() {
|
|
|
93
93
|
return () => clearInterval(timer);
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
export { startIdleHeapCleanup };
|
|
96
|
+
export { collectAndReturnPages, startIdleHeapCleanup };
|
|
@@ -1,20 +1,24 @@
|
|
|
1
|
-
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-
|
|
1
|
+
import { validateWorkerConfig, validateIndexName, validatePatternValueLimit, requireSurfaceFormsForPartialFields, validateBM25Params, validatePartitionConfig, executeQuery, queryBindingOf, executePreflight, executeSuggest, executeListDocuments, validateDocId } from './chunk-4D6OH4EF.mjs';
|
|
2
2
|
import { ReplicationMessageTypes } from './chunk-GCM3JFVA.mjs';
|
|
3
|
-
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-
|
|
4
|
-
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION,
|
|
5
|
-
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-
|
|
6
|
-
import { detectRuntime, getLanguage } from './chunk-
|
|
7
|
-
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-
|
|
8
|
-
import { releaseLocksHeldBy } from './chunk-
|
|
9
|
-
import { requireSearchBackendSettings } from './chunk-
|
|
10
|
-
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-
|
|
11
|
-
import { getNestedValue, parseVectorDimension } from './chunk-
|
|
12
|
-
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-
|
|
3
|
+
import { WIRE_BATCH_BUDGET, resolveVectorText, bindEngineCore, restoredConfigFields, restoredEmbedding, collectRequiredFieldFailures, embedChunkDocuments, providedDocId, requireDocumentObject, admitInsert, asBatchInsertError, requireMatchingDocId, embedDocumentFields, assertDocumentCarriesMappedVectors, admitBufferedUpsert, createPendingAdmissions, recordPendingAdmission } from './chunk-2453RJYK.mjs';
|
|
4
|
+
import { NATIVE_CRC32_MIN_BYTES, generateId, decodeEveryStoredDocument, createSharedFrozenSegment, shareSegmentArrays, splitWorkerBudget, resolveWorkerCount, createDirectExecutor, createPartitionRouter, MAX_FILE_BYTES, resolveWorkerEntry, missingWorkerEntry, readHeapStatistics, nodeThreadId, CHECKSUM_YIELD_CHUNK_BYTES, resolvePartitionInsertOptions, resolveIndexAnalysis, createPartitionManager, buildSegmentPayload, createFrozenSegment, resolveRequestThreadCount, vectorsPerFile, vectorsToBytes, VECTOR_FILE_PAYLOAD_VERSION, createPartitionIndex, selectionKeeps, decodeMapWithoutFields, isCompositePartition, segmentArraysOf, mergeSegmentContents, survivorDocumentsOf, partitionStateOf, VECTOR_FILE_MAX_VECTORS, deadBytesFor, mergeFrozenSegments, workerResourceLimits, decodeVectorIndexParts, bytesToVectors, freezeSegmentShared, DEFAULT_BACKPRESSURE_LIMIT, DEFAULT_REQUEST_TIMEOUT_MS, SHUTDOWN_TIMEOUT_MS, decodeVectorFilePayload, decodeVectorGraphPartPayload, assembleGraphParts, countDead, encodeSegmentState, CONSECUTIVE_TIMEOUTS_BEFORE_DEATH, VALID_HNSW_METRICS, spawnNodeWorker, decodeVectorGraphPayload, graphPartsOfEarlierPayload, decodeStoredDocument, isDead } from './chunk-CUP2E4OH.mjs';
|
|
5
|
+
import { requirePatternSearchFor, requirePartialWordSearchFor, collectQueryTermStats, mergePartitionStats, oversampledFacetConfig, fnv1a, availablePatternSearch, PATTERN_HOST, kWayMerge, mergeFacets, setNestedValue } from './chunk-XYDP7CCJ.mjs';
|
|
6
|
+
import { detectRuntime, getLanguage } from './chunk-OLXM65YH.mjs';
|
|
7
|
+
import { MAX_INDEX_CONFIGURATION_BYTES, STATISTICS_BROADCAST_INTERVAL_MS, MAX_SWEEP_INTERVAL_MS, MIN_SWEEP_INTERVAL_MS, HEAP_PRESSURE_WARNING_FRACTION, MAX_EXPORT_LINE_BYTES, IMPORT_BATCH_DOCUMENTS, FOREIGN_STATS_TTL_MS, MAX_IMPORT_REFUSALS_REPORTED, BATCH_CHUNK_SIZE, MIN_DOCUMENTS_FOR_SEGMENTS, HEAP_PRESSURE_REARM_FRACTION, EXPORT_CHUNK_BYTES } from './chunk-ZCKPRCYI.mjs';
|
|
8
|
+
import { releaseLocksHeldBy } from './chunk-RZXNWQSM.mjs';
|
|
9
|
+
import { requireSearchBackendSettings } from './chunk-B3UOQJD3.mjs';
|
|
10
|
+
import { DEFAULT_PATTERN_WORK_CAP, MATCHER_STATE_BUDGET_BYTES } from './chunk-SREPGC6O.mjs';
|
|
11
|
+
import { getNestedValue, parseVectorDimension } from './chunk-4VBG6PEH.mjs';
|
|
12
|
+
import { extractVectorFieldsFromSchema, messagePackUpperBoundBytes, validateSchema, requireCreatableSchema, validateVectorPromotion, validateVectorStorage, raiseEarlierReleaseGraphSetting, partialFieldsOf, flattenSchema, requireValidFieldTypes, encodeDocument, schemaFieldsOf, isQuantizationMode, isStorageMode, validateRequiredFields, requireDocumentWithinSizeLimit, validateDocument, validateDocumentStrict } from './chunk-KJ7LB3TR.mjs';
|
|
13
13
|
import { MIN_M } from './chunk-KEHK2O6P.mjs';
|
|
14
|
-
import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-
|
|
14
|
+
import { isSingleStringFieldType, VECTOR_PATTERN, MAX_DOCUMENT_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
15
15
|
import { compareCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
16
16
|
import { createRequestId, isValidWorkerResponse, threadSlotOfWorker } from './chunk-POWOTUKJ.mjs';
|
|
17
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
17
|
+
import { ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
18
|
+
import { openSync, close, readFileSync } from 'fs';
|
|
19
|
+
import { rename, readFile, writeFile, unlink, realpath, readdir, readlink } from 'fs/promises';
|
|
20
|
+
import { endianness } from 'os';
|
|
21
|
+
import { dirname } from 'path';
|
|
18
22
|
import { encode, decode } from '@msgpack/msgpack';
|
|
19
23
|
import * as zlib from 'zlib';
|
|
20
24
|
|
|
@@ -728,15 +732,48 @@ async function getPath() {
|
|
|
728
732
|
}
|
|
729
733
|
return cachedPath;
|
|
730
734
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
var
|
|
734
|
-
var
|
|
735
|
-
var
|
|
735
|
+
var AUXV_CLOCK_TICKS_TYPE = 17n;
|
|
736
|
+
var AUXV_END_TYPE = 0n;
|
|
737
|
+
var THIRTY_TWO_BIT_ARCHITECTURES = /* @__PURE__ */ new Set(["arm", "ia32", "mips", "mipsel", "ppc"]);
|
|
738
|
+
var FIRST_STAT_FIELD_AFTER_COMMAND = 3;
|
|
739
|
+
var STAT_STATE_FIELD = 3;
|
|
740
|
+
var STAT_START_TIME_FIELD = 22;
|
|
741
|
+
var EXITED_STATES = /* @__PURE__ */ new Set(["Z", "X", "x"]);
|
|
742
|
+
var UNKNOWN = { kind: "unknown" };
|
|
743
|
+
var clockTicksPerSecond = null;
|
|
736
744
|
function errnoOf(err) {
|
|
737
745
|
return err?.code;
|
|
738
746
|
}
|
|
739
|
-
function
|
|
747
|
+
function readWord(bytes, offset, wordBytes, littleEndian) {
|
|
748
|
+
if (wordBytes === 8) return littleEndian ? bytes.readBigUInt64LE(offset) : bytes.readBigUInt64BE(offset);
|
|
749
|
+
return BigInt(littleEndian ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset));
|
|
750
|
+
}
|
|
751
|
+
async function readClockTicksPerSecond() {
|
|
752
|
+
const auxv = await readFile("/proc/self/auxv");
|
|
753
|
+
const wordBytes = THIRTY_TWO_BIT_ARCHITECTURES.has(process.arch) ? 4 : 8;
|
|
754
|
+
const littleEndian = endianness() === "LE";
|
|
755
|
+
for (let offset = 0; offset + 2 * wordBytes <= auxv.length; offset += 2 * wordBytes) {
|
|
756
|
+
const type = readWord(auxv, offset, wordBytes, littleEndian);
|
|
757
|
+
if (type === AUXV_END_TYPE) return null;
|
|
758
|
+
if (type === AUXV_CLOCK_TICKS_TYPE) {
|
|
759
|
+
const ticks = Number(readWord(auxv, offset + wordBytes, wordBytes, littleEndian));
|
|
760
|
+
return ticks > 0 ? ticks : null;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
return null;
|
|
764
|
+
}
|
|
765
|
+
function clockTicks() {
|
|
766
|
+
clockTicksPerSecond ??= readClockTicksPerSecond().catch(() => null);
|
|
767
|
+
return clockTicksPerSecond;
|
|
768
|
+
}
|
|
769
|
+
function threadGroupOf(status) {
|
|
770
|
+
const leader = /^Tgid:\s*(\d+)\s*$/m.exec(status)?.[1];
|
|
771
|
+
return leader === void 0 ? null : Number(leader);
|
|
772
|
+
}
|
|
773
|
+
function statField(fieldsAfterCommand, field) {
|
|
774
|
+
return fieldsAfterCommand[field - FIRST_STAT_FIELD_AFTER_COMMAND];
|
|
775
|
+
}
|
|
776
|
+
function signalReaches(pid) {
|
|
740
777
|
try {
|
|
741
778
|
process.kill(pid, 0);
|
|
742
779
|
return true;
|
|
@@ -744,6 +781,101 @@ function processIsRunning(pid) {
|
|
|
744
781
|
return errnoOf(err) === "EPERM";
|
|
745
782
|
}
|
|
746
783
|
}
|
|
784
|
+
async function identifyProcess(pid) {
|
|
785
|
+
try {
|
|
786
|
+
const [status, stat, ticksPerSecond] = await Promise.all([
|
|
787
|
+
readFile(`/proc/${pid}/status`, "utf8"),
|
|
788
|
+
readFile(`/proc/${pid}/stat`, "utf8"),
|
|
789
|
+
clockTicks()
|
|
790
|
+
]);
|
|
791
|
+
const leader = threadGroupOf(status);
|
|
792
|
+
if (leader === null) return UNKNOWN;
|
|
793
|
+
if (leader !== pid) return { kind: "thread" };
|
|
794
|
+
const commandEnd = stat.lastIndexOf(")");
|
|
795
|
+
if (commandEnd < 0) return UNKNOWN;
|
|
796
|
+
const fieldsAfterCommand = stat.slice(commandEnd + 1).trim().split(/\s+/);
|
|
797
|
+
const state = statField(fieldsAfterCommand, STAT_STATE_FIELD);
|
|
798
|
+
if (state !== void 0 && EXITED_STATES.has(state)) return { kind: "exited" };
|
|
799
|
+
const startTicks = Number(statField(fieldsAfterCommand, STAT_START_TIME_FIELD));
|
|
800
|
+
const uptimeSeconds = Number.parseFloat(readFileSync("/proc/uptime", "utf8"));
|
|
801
|
+
const nowMs = Date.now();
|
|
802
|
+
if (ticksPerSecond === null || !Number.isFinite(startTicks) || !Number.isFinite(uptimeSeconds)) return UNKNOWN;
|
|
803
|
+
return { kind: "process", startMs: Math.round(nowMs - (uptimeSeconds - startTicks / ticksPerSecond) * 1e3) };
|
|
804
|
+
} catch {
|
|
805
|
+
return UNKNOWN;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
async function descriptorTargets(pid) {
|
|
809
|
+
const descriptorDirectory = `/proc/${pid}/fd`;
|
|
810
|
+
let descriptors;
|
|
811
|
+
try {
|
|
812
|
+
descriptors = await readdir(descriptorDirectory);
|
|
813
|
+
} catch {
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
816
|
+
const targets = [];
|
|
817
|
+
for (const descriptor of descriptors) {
|
|
818
|
+
try {
|
|
819
|
+
targets.push(await readlink(`${descriptorDirectory}/${descriptor}`));
|
|
820
|
+
} catch (err) {
|
|
821
|
+
if (errnoOf(err) !== "ENOENT") return null;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
return targets;
|
|
825
|
+
}
|
|
826
|
+
async function holdsFileInDirectoryOf(pid, lockPath) {
|
|
827
|
+
const targets = await descriptorTargets(pid);
|
|
828
|
+
if (targets === null) return false;
|
|
829
|
+
const directory = await realpath(dirname(lockPath)).catch(() => null);
|
|
830
|
+
if (directory === null) return false;
|
|
831
|
+
const inside = directory.endsWith("/") ? directory : `${directory}/`;
|
|
832
|
+
return targets.some((target) => target === directory || target.startsWith(inside));
|
|
833
|
+
}
|
|
834
|
+
async function holderIsLive(holder, self, lockPath) {
|
|
835
|
+
if (holder.pid === self.pid) {
|
|
836
|
+
if (holder.processStartMs === null || self.processStartMs === null) return false;
|
|
837
|
+
if (Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS) return true;
|
|
838
|
+
return holdsFileInDirectoryOf(holder.pid, lockPath);
|
|
839
|
+
}
|
|
840
|
+
if (!signalReaches(holder.pid)) return false;
|
|
841
|
+
const found = await identifyProcess(holder.pid);
|
|
842
|
+
if (found.kind === "thread" || found.kind === "exited") return false;
|
|
843
|
+
if (found.kind === "unknown") return signalReaches(holder.pid);
|
|
844
|
+
if (holder.processStartMs === null) return true;
|
|
845
|
+
if (found.startMs - holder.processStartMs <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS) return true;
|
|
846
|
+
return holdsFileInDirectoryOf(holder.pid, lockPath);
|
|
847
|
+
}
|
|
848
|
+
function holdDirectoryOpen(directory) {
|
|
849
|
+
if (process.platform !== "linux") return () => void 0;
|
|
850
|
+
const descriptor = openSync(directory, "r");
|
|
851
|
+
return () => close(descriptor, () => void 0);
|
|
852
|
+
}
|
|
853
|
+
async function removeStaleLock(lockPath, asidePath, isStale) {
|
|
854
|
+
try {
|
|
855
|
+
await rename(lockPath, asidePath);
|
|
856
|
+
} catch (err) {
|
|
857
|
+
if (errnoOf(err) === "ENOENT") return;
|
|
858
|
+
throw err;
|
|
859
|
+
}
|
|
860
|
+
try {
|
|
861
|
+
const moved = await readFile(asidePath, "utf8");
|
|
862
|
+
if (!isStale(moved)) {
|
|
863
|
+
await writeFile(lockPath, moved, { flag: "wx" }).catch((err) => {
|
|
864
|
+
if (errnoOf(err) !== "EEXIST") throw err;
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
} finally {
|
|
868
|
+
await unlink(asidePath).catch(() => void 0);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
// src/persistence/durability/directory-lock.ts
|
|
873
|
+
var LOCK_FILE_NAME = ".narsil.lock";
|
|
874
|
+
var HARD_LINK_UNSUPPORTED_CODES = /* @__PURE__ */ new Set(["EPERM", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]);
|
|
875
|
+
var lockedDirectories = /* @__PURE__ */ new Set();
|
|
876
|
+
function errnoOf2(err) {
|
|
877
|
+
return err?.code;
|
|
878
|
+
}
|
|
747
879
|
function integerOrNull(text) {
|
|
748
880
|
const value = Number.parseInt(text ?? "", 10);
|
|
749
881
|
return Number.isSafeInteger(value) ? value : null;
|
|
@@ -771,7 +903,7 @@ async function readHolder(fs, lockPath) {
|
|
|
771
903
|
try {
|
|
772
904
|
return parseHolder(await fs.readFile(lockPath, "utf8")) ?? "unreadable";
|
|
773
905
|
} catch (err) {
|
|
774
|
-
if (
|
|
906
|
+
if (errnoOf2(err) === "ENOENT") return "absent";
|
|
775
907
|
throw err;
|
|
776
908
|
}
|
|
777
909
|
}
|
|
@@ -781,10 +913,9 @@ async function settledHolder(fs, lockPath) {
|
|
|
781
913
|
await new Promise((resolve) => setTimeout(resolve, DIRECTORY_LOCK_SETTLE_MS));
|
|
782
914
|
return readHolder(fs, lockPath);
|
|
783
915
|
}
|
|
784
|
-
function
|
|
785
|
-
if (
|
|
786
|
-
|
|
787
|
-
return Math.abs(holder.processStartMs - self.processStartMs) <= DIRECTORY_LOCK_PROCESS_START_TOLERANCE_MS;
|
|
916
|
+
function sameReading(first, second) {
|
|
917
|
+
if (typeof first !== "object" || typeof second !== "object") return first === second;
|
|
918
|
+
return formatHolder(first) === formatHolder(second);
|
|
788
919
|
}
|
|
789
920
|
function heldElsewhere(root, holder, lockPath) {
|
|
790
921
|
return new NarsilError(
|
|
@@ -802,7 +933,7 @@ async function createLockFileExclusively(fs, lockPath, content) {
|
|
|
802
933
|
await fs.writeFile(lockPath, content, { flag: "wx" });
|
|
803
934
|
return true;
|
|
804
935
|
} catch (err) {
|
|
805
|
-
if (
|
|
936
|
+
if (errnoOf2(err) === "EEXIST") return false;
|
|
806
937
|
throw err;
|
|
807
938
|
}
|
|
808
939
|
}
|
|
@@ -812,7 +943,7 @@ async function createLockFile(fs, lockPath, draftPath, content) {
|
|
|
812
943
|
await fs.link(draftPath, lockPath);
|
|
813
944
|
return true;
|
|
814
945
|
} catch (err) {
|
|
815
|
-
const code =
|
|
946
|
+
const code = errnoOf2(err);
|
|
816
947
|
if (code === "EEXIST") return false;
|
|
817
948
|
if (code !== void 0 && HARD_LINK_UNSUPPORTED_CODES.has(code)) {
|
|
818
949
|
return createLockFileExclusively(fs, lockPath, content);
|
|
@@ -828,12 +959,14 @@ async function acquire(fs, root, lockPath, self) {
|
|
|
828
959
|
for (let attempt = 0; attempt < DIRECTORY_LOCK_ATTEMPTS; attempt++) {
|
|
829
960
|
if (await createLockFile(fs, lockPath, draftPath, content)) return;
|
|
830
961
|
const holder = await settledHolder(fs, lockPath);
|
|
831
|
-
if (typeof holder === "object" && holderIsLive(holder, self)) {
|
|
962
|
+
if (typeof holder === "object" && await holderIsLive(holder, self, lockPath)) {
|
|
832
963
|
throw heldElsewhere(root, holderLabel(holder, self), lockPath);
|
|
833
964
|
}
|
|
834
|
-
await
|
|
835
|
-
|
|
836
|
-
|
|
965
|
+
await removeStaleLock(
|
|
966
|
+
lockPath,
|
|
967
|
+
`${draftPath}.aside`,
|
|
968
|
+
(text) => sameReading(holder, parseHolder(text) ?? "unreadable")
|
|
969
|
+
);
|
|
837
970
|
}
|
|
838
971
|
throw heldElsewhere(root, "another process", lockPath);
|
|
839
972
|
}
|
|
@@ -851,7 +984,15 @@ async function lockDirectory(root) {
|
|
|
851
984
|
const resolvedRoot = pathMod.resolve(root);
|
|
852
985
|
const lockPath = pathMod.join(resolvedRoot, LOCK_FILE_NAME);
|
|
853
986
|
const self = await describeSelf();
|
|
987
|
+
async function holds() {
|
|
988
|
+
const holder = await readHolder(fs, lockPath);
|
|
989
|
+
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
990
|
+
}
|
|
991
|
+
async function removeOwnLockFile() {
|
|
992
|
+
if (await holds().catch(() => false)) await fs.unlink(lockPath).catch(() => void 0);
|
|
993
|
+
}
|
|
854
994
|
let directoryKey = null;
|
|
995
|
+
let closeDirectory = () => void 0;
|
|
855
996
|
try {
|
|
856
997
|
await fs.mkdir(resolvedRoot, { recursive: true });
|
|
857
998
|
const stat = await fs.stat(resolvedRoot, { bigint: true });
|
|
@@ -860,8 +1001,10 @@ async function lockDirectory(root) {
|
|
|
860
1001
|
lockedDirectories.add(key);
|
|
861
1002
|
directoryKey = key;
|
|
862
1003
|
await acquire(fs, root, lockPath, self);
|
|
1004
|
+
closeDirectory = holdDirectoryOpen(resolvedRoot);
|
|
863
1005
|
} catch (err) {
|
|
864
1006
|
if (directoryKey !== null) lockedDirectories.delete(directoryKey);
|
|
1007
|
+
await removeOwnLockFile();
|
|
865
1008
|
if (err instanceof NarsilError) throw err;
|
|
866
1009
|
throw new NarsilError(
|
|
867
1010
|
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
@@ -871,17 +1014,14 @@ async function lockDirectory(root) {
|
|
|
871
1014
|
}
|
|
872
1015
|
const heldKey = directoryKey;
|
|
873
1016
|
let released = false;
|
|
874
|
-
async function holds() {
|
|
875
|
-
const holder = await readHolder(fs, lockPath);
|
|
876
|
-
return typeof holder === "object" && holder.nonce === self.nonce;
|
|
877
|
-
}
|
|
878
1017
|
return {
|
|
879
1018
|
holds,
|
|
880
1019
|
async release() {
|
|
881
1020
|
if (released) return;
|
|
882
1021
|
released = true;
|
|
883
1022
|
lockedDirectories.delete(heldKey);
|
|
884
|
-
|
|
1023
|
+
await removeOwnLockFile();
|
|
1024
|
+
closeDirectory();
|
|
885
1025
|
}
|
|
886
1026
|
};
|
|
887
1027
|
}
|
|
@@ -2989,6 +3129,13 @@ function createInsyncConfirmMessage(payload, sourceId, requestId) {
|
|
|
2989
3129
|
function isRecord2(value) {
|
|
2990
3130
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2991
3131
|
}
|
|
3132
|
+
function isPlainObject(value) {
|
|
3133
|
+
if (!isRecord2(value)) {
|
|
3134
|
+
return false;
|
|
3135
|
+
}
|
|
3136
|
+
const prototype = Object.getPrototypeOf(value);
|
|
3137
|
+
return prototype === Object.prototype || prototype === null;
|
|
3138
|
+
}
|
|
2992
3139
|
function validateEntryPayload(decoded) {
|
|
2993
3140
|
if (!isRecord2(decoded) || !isRecord2(decoded.entry)) {
|
|
2994
3141
|
throw new Error('Invalid EntryPayload: missing or invalid "entry" field');
|
|
@@ -3110,6 +3257,9 @@ function validateForwardBatchResultPayload(decoded) {
|
|
|
3110
3257
|
if (result.errorMessage !== null && typeof result.errorMessage !== "string") {
|
|
3111
3258
|
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorMessage" must be a string or null`);
|
|
3112
3259
|
}
|
|
3260
|
+
if (result.errorDetails !== void 0 && result.errorDetails !== null && !isPlainObject(result.errorDetails)) {
|
|
3261
|
+
throw new Error(`Invalid ForwardBatchResultPayload: "results[${index}].errorDetails" must be an object or absent`);
|
|
3262
|
+
}
|
|
3113
3263
|
}
|
|
3114
3264
|
return decoded;
|
|
3115
3265
|
}
|
|
@@ -3606,7 +3756,8 @@ async function readVectorFile(store, file, fieldPath) {
|
|
|
3606
3756
|
async function loadVectorField(store, ref, vectorIndex) {
|
|
3607
3757
|
const restore = vectorIndex.restoreCheckpoint({
|
|
3608
3758
|
liveVectors: liveVectorsOf(ref),
|
|
3609
|
-
holdsGraph: ref.graphParts.length > 0
|
|
3759
|
+
holdsGraph: ref.graphParts.length > 0,
|
|
3760
|
+
graph: await readGraphParts(store, ref)
|
|
3610
3761
|
});
|
|
3611
3762
|
for (const file of ref.files) {
|
|
3612
3763
|
const { payload, payloadLength } = await readVectorFile(store, file, ref.fieldPath);
|
|
@@ -3621,7 +3772,7 @@ async function loadVectorField(store, ref, vectorIndex) {
|
|
|
3621
3772
|
}
|
|
3622
3773
|
});
|
|
3623
3774
|
}
|
|
3624
|
-
restore.finish(
|
|
3775
|
+
restore.finish();
|
|
3625
3776
|
}
|
|
3626
3777
|
var EMPTY_BIN = new Uint8Array(0);
|
|
3627
3778
|
var EMPTY_BIN_BYTES = 2;
|
|
@@ -3685,7 +3836,7 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
3685
3836
|
const files = plan.kept.map((kept) => keptFileRef(input, prior, kept));
|
|
3686
3837
|
const written = [];
|
|
3687
3838
|
for (let index = 0; index < plan.newFiles; index++) {
|
|
3688
|
-
const { payload, ordinals } = plan.readNewFile(index);
|
|
3839
|
+
const { payload, ordinals, dead } = plan.readNewFile(index);
|
|
3689
3840
|
const id = nextFileId;
|
|
3690
3841
|
nextFileId += 1;
|
|
3691
3842
|
const key = vectorFileKey(input.indexName, fieldPath, id);
|
|
@@ -3695,13 +3846,13 @@ async function writeField(input, fieldPath, plan, outcome) {
|
|
|
3695
3846
|
`vector file "${key}"`
|
|
3696
3847
|
);
|
|
3697
3848
|
await input.store.write(key, packed.chunks);
|
|
3698
|
-
const dead = deadBitsWhere(ordinals.length, (position) => ordinals[position] === NO_LIVE_VECTOR_AT_POSITION);
|
|
3699
3849
|
files.push({ id, key, count: payload.docIds.length, dead, bytes: packed.byteLength, checksum: packed.checksum });
|
|
3700
3850
|
written.push({ id, key, ordinals });
|
|
3701
3851
|
outcome.layouts.push({
|
|
3702
3852
|
fieldPath,
|
|
3703
3853
|
key,
|
|
3704
3854
|
docIds: payload.docIds,
|
|
3855
|
+
dead,
|
|
3705
3856
|
vectorsOffset: packed.byteLength - payload.vectors.byteLength
|
|
3706
3857
|
});
|
|
3707
3858
|
}
|
|
@@ -5572,7 +5723,12 @@ async function adoptVectorLayouts(store, vectorIndexes, layouts) {
|
|
|
5572
5723
|
const vecIndex = vectorIndexes.get(layout.fieldPath);
|
|
5573
5724
|
if (vecIndex === void 0 || vecIndex.storage !== "disk") continue;
|
|
5574
5725
|
const path = await pathOf.call(store, layout.key);
|
|
5575
|
-
await vecIndex.adoptDiskLayout({
|
|
5726
|
+
await vecIndex.adoptDiskLayout({
|
|
5727
|
+
path,
|
|
5728
|
+
docIds: layout.docIds,
|
|
5729
|
+
dead: layout.dead,
|
|
5730
|
+
vectorsOffset: layout.vectorsOffset
|
|
5731
|
+
});
|
|
5576
5732
|
}
|
|
5577
5733
|
}
|
|
5578
5734
|
async function garbageThatNoFieldStillReads(store, vectorIndexes, keys) {
|
|
@@ -6527,7 +6683,11 @@ async function readGraph(store, ref) {
|
|
|
6527
6683
|
async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
6528
6684
|
let liveVectors = 0;
|
|
6529
6685
|
for (const file of ref.files) liveVectors += file.count - countDead(file.dead, file.count);
|
|
6530
|
-
const restore = vectorIndex.restoreCheckpoint({
|
|
6686
|
+
const restore = vectorIndex.restoreCheckpoint({
|
|
6687
|
+
liveVectors,
|
|
6688
|
+
holdsGraph: ref.graphKey !== null,
|
|
6689
|
+
graph: await readGraph(store, ref)
|
|
6690
|
+
});
|
|
6531
6691
|
for (const file of ref.files) {
|
|
6532
6692
|
const bytes = await store.read(file.key);
|
|
6533
6693
|
if (bytes === null) {
|
|
@@ -6560,7 +6720,7 @@ async function loadVectorField2(store, ref, vectorIndex, fileRefs) {
|
|
|
6560
6720
|
}
|
|
6561
6721
|
});
|
|
6562
6722
|
}
|
|
6563
|
-
restore.finish(
|
|
6723
|
+
restore.finish();
|
|
6564
6724
|
}
|
|
6565
6725
|
async function loadEarlierLayout(store, indexName, manifest, deps) {
|
|
6566
6726
|
let highestPartitionId = manifest.partitions.length - 1;
|
|
@@ -7366,7 +7526,7 @@ function createDurabilityManager(config, hooks, directoryOverride) {
|
|
|
7366
7526
|
if (indexState === void 0) {
|
|
7367
7527
|
return;
|
|
7368
7528
|
}
|
|
7369
|
-
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), (
|
|
7529
|
+
await drainIndexStateForUnload(indexState, metadataWrites.get(indexName), (close2) => close2);
|
|
7370
7530
|
indexes.delete(indexName);
|
|
7371
7531
|
metadataWrites.delete(indexName);
|
|
7372
7532
|
},
|
|
@@ -13540,7 +13700,7 @@ function partitionVectorPlan(vectorIndex, docIds) {
|
|
|
13540
13700
|
codes: null,
|
|
13541
13701
|
vectors: vectorsToBytes(packed)
|
|
13542
13702
|
};
|
|
13543
|
-
files.push({ payload, ordinals: new Int32Array(slice.length) });
|
|
13703
|
+
files.push({ payload, ordinals: new Int32Array(slice.length), dead: null });
|
|
13544
13704
|
}
|
|
13545
13705
|
return {
|
|
13546
13706
|
unchanged: false,
|
|
@@ -14068,17 +14228,17 @@ function rejectConfiguration(message) {
|
|
|
14068
14228
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid index configuration: ${message}`);
|
|
14069
14229
|
}
|
|
14070
14230
|
function renamed(manifest, sourceName, indexName) {
|
|
14071
|
-
const
|
|
14231
|
+
const rename2 = (key) => `${indexName}/${key.slice(sourceName.length + 1)}`;
|
|
14072
14232
|
return {
|
|
14073
14233
|
...manifest,
|
|
14074
14234
|
partitions: manifest.partitions.map((partition) => ({
|
|
14075
14235
|
...partition,
|
|
14076
|
-
segments: partition.segments.map((segment) => ({ ...segment, key:
|
|
14236
|
+
segments: partition.segments.map((segment) => ({ ...segment, key: rename2(segment.key) }))
|
|
14077
14237
|
})),
|
|
14078
14238
|
vectors: manifest.vectors.map((field) => ({
|
|
14079
14239
|
...field,
|
|
14080
|
-
files: field.files.map((file) => ({ ...file, key:
|
|
14081
|
-
graphParts: field.graphParts.map((part) => ({ ...part, key:
|
|
14240
|
+
files: field.files.map((file) => ({ ...file, key: rename2(file.key) })),
|
|
14241
|
+
graphParts: field.graphParts.map((part) => ({ ...part, key: rename2(part.key) }))
|
|
14082
14242
|
}))
|
|
14083
14243
|
};
|
|
14084
14244
|
}
|