@delali/narsil 0.3.1-next.86 → 0.3.1-next.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/chunk-3BXQPIXK.mjs +602 -0
- package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
- package/dist/browser/chunk-BBDAYUCO.mjs +2703 -0
- package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
- package/dist/browser/chunk-TBMNPGFO.mjs +22910 -0
- package/dist/browser/durability-wiring-7RUN7LNS.mjs +3250 -0
- package/dist/browser/module-E25UDADM.mjs +2766 -0
- package/dist/browser/snapshot-A42DRERF.mjs +888 -0
- package/dist/browser/write-4U6Q26TJ.mjs +206 -0
- package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-AQU2QHSY.mjs +7247 -0
- package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
- package/dist/{chunk-4D6OH4EF.mjs → chunk-AXDAS2VA.mjs} +7 -5
- package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-FGMGUDHX.mjs +21 -0
- package/dist/chunk-IKST36MA.mjs +614 -0
- package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
- package/dist/chunk-JFJBA3PD.mjs +522 -0
- package/dist/chunk-K4OWQ2M3.mjs +462 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/chunk-MKK4OZ5W.mjs +703 -0
- package/dist/chunk-MXS5HWIH.mjs +515 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PR4O7CAP.mjs +18 -0
- package/dist/{chunk-4VBG6PEH.mjs → chunk-PSUJCOSE.mjs} +3 -2
- package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
- package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
- package/dist/chunk-R7OSDAFK.mjs +44 -0
- package/dist/chunk-RVM7AODE.mjs +73 -0
- package/dist/{chunk-XYDP7CCJ.mjs → chunk-SYRFSF6V.mjs} +6 -4
- package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
- package/dist/chunk-T7DMUVTL.mjs +1714 -0
- package/dist/chunk-UKSUENHK.mjs +997 -0
- package/dist/{chunk-B3F7LZXM.mjs → chunk-ULRPNYAG.mjs} +1220 -2318
- package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
- package/dist/chunk-WQSHDZ7Z.mjs +631 -0
- package/dist/{chunk-XPK3GB2P.mjs → chunk-XBXKPWMR.mjs} +20 -2664
- package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
- package/dist/chunk-Y2F66IQJ.mjs +74 -0
- package/dist/chunk-YCJLVNWR.mjs +262 -0
- package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
- package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
- package/dist/direct-executor-LD6JW57R.mjs +18 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +30 -15
- package/dist/durability-wiring-5S32KDDC.mjs +2797 -0
- package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
- package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
- package/dist/index.browser.mjs +2 -38128
- package/dist/index.d.ts +23 -9
- package/dist/index.mjs +18 -14
- package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
- package/dist/module-WIDYIJLY.mjs +1819 -0
- package/dist/partial-words.mjs +5 -3
- package/dist/pattern.mjs +3 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
- package/dist/react.d.ts +7 -7
- package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +14 -9
- package/dist/snapshot-6BYSUPWL.mjs +469 -0
- package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
- package/dist/{thread-EUUNJNKU.mjs → thread-GS65E6RM.mjs} +12 -9
- package/dist/vector/search-worker.d.ts +30 -3
- package/dist/vector/search-worker.mjs +6 -2
- package/dist/workers/entry.mjs +5 -3
- package/dist/write-HLEZTL2L.mjs +233 -0
- package/package.json +11 -7
- package/dist/chunk-IDLP54FH.mjs +0 -15198
- package/dist/direct-executor-SI2QV4YI.mjs +0 -19
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
|
@@ -1,18 +1,21 @@
|
|
|
1
|
-
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
4
|
-
import './chunk-
|
|
1
|
+
import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-XSZVS4S3.mjs';
|
|
2
|
+
import './chunk-DJHHF2L5.mjs';
|
|
3
|
+
import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-AXDAS2VA.mjs';
|
|
4
|
+
import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-SYRFSF6V.mjs';
|
|
5
|
+
import './chunk-Q5DC6WAB.mjs';
|
|
5
6
|
import './chunk-ZJ4CSAB2.mjs';
|
|
6
|
-
import './chunk-
|
|
7
|
-
import './chunk-
|
|
7
|
+
import './chunk-QZN3MIJ3.mjs';
|
|
8
|
+
import './chunk-T6CNO72J.mjs';
|
|
8
9
|
import './chunk-WQBCBKY3.mjs';
|
|
9
|
-
import './chunk-
|
|
10
|
-
import { extractVectorFieldsFromSchema } from './chunk-
|
|
10
|
+
import './chunk-PSUJCOSE.mjs';
|
|
11
|
+
import { extractVectorFieldsFromSchema } from './chunk-CVQWQVFX.mjs';
|
|
12
|
+
import './chunk-FGMGUDHX.mjs';
|
|
11
13
|
import './chunk-KEHK2O6P.mjs';
|
|
12
14
|
import './chunk-GES7U2PM.mjs';
|
|
13
|
-
import './chunk-
|
|
15
|
+
import './chunk-BVOJIYRV.mjs';
|
|
14
16
|
import './chunk-R6A773WK.mjs';
|
|
15
17
|
import './chunk-YTJCGKM4.mjs';
|
|
18
|
+
import './chunk-L5ORED7V.mjs';
|
|
16
19
|
import { ServerErrorCodes, ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
|
|
17
20
|
|
|
18
21
|
// src/server/request-threads/held-vectors.ts
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import { e as VectorQuantizationMode } from '../schema-
|
|
2
|
-
import { A as ArenaSimd } from '../simd-DW5AtJ6B.js';
|
|
1
|
+
import { e as VectorQuantizationMode } from '../schema-DtsgmNfU.js';
|
|
3
2
|
import '../adapters-79OIa4xm.js';
|
|
4
3
|
|
|
5
4
|
/**
|
|
@@ -42,6 +41,34 @@ interface OrdinalFilter {
|
|
|
42
41
|
*/
|
|
43
42
|
type GrowableBuffer = SharedArrayBuffer | ArrayBuffer;
|
|
44
43
|
|
|
44
|
+
/**
|
|
45
|
+
* The distance kernels bound to one memory, which read their operands at byte
|
|
46
|
+
* offsets inside that memory so that a caller measures against a vector where
|
|
47
|
+
* it already lies.
|
|
48
|
+
*
|
|
49
|
+
* @internal
|
|
50
|
+
*/
|
|
51
|
+
interface ArenaSimd {
|
|
52
|
+
/** The kernels read every operand out of this memory. */
|
|
53
|
+
memory: WebAssembly.Memory;
|
|
54
|
+
/** Returns the dot product of two float32 vectors at the given byte offsets. */
|
|
55
|
+
dot_product: (ptrA: number, ptrB: number, len: number) => number;
|
|
56
|
+
/** Returns the length of the float32 vector at the given byte offset. */
|
|
57
|
+
magnitude: (ptr: number, len: number) => number;
|
|
58
|
+
/** Returns the squared euclidean distance between two float32 vectors at the given byte offsets. */
|
|
59
|
+
squared_euclidean_distance: (ptrA: number, ptrB: number, len: number) => number;
|
|
60
|
+
/** Returns the integer dot product of two byte code vectors at the given byte offsets. */
|
|
61
|
+
dot_u8: (ptrA: number, ptrB: number, len: number) => number;
|
|
62
|
+
/** Returns the integer squared distance between two byte code vectors at the given byte offsets. */
|
|
63
|
+
sqdist_u8: (ptrA: number, ptrB: number, len: number) => number;
|
|
64
|
+
/** Returns the sum of level products of two 4-bit codes at the given byte offsets, each holding two levels per byte. */
|
|
65
|
+
osq_dot_nibbles_4x4: (ptrA: number, ptrB: number, bytes: number) => number;
|
|
66
|
+
/** Returns the sum of level products of a 1-bit or 2-bit code and a query staged as four bit planes. Each plane spans `planeBytes`, with one bit for each of the code's lowest level bits. */
|
|
67
|
+
osq_dot_bits: (ptrDoc: number, ptrPlanes: number, planeBytes: number, docBits: number) => number;
|
|
68
|
+
/** Returns the sum of level products of two 1-bit codes or two 2-bit codes at the given byte offsets. */
|
|
69
|
+
osq_dot_bits_pair: (ptrA: number, ptrB: number, bytes: number, bits: number) => number;
|
|
70
|
+
}
|
|
71
|
+
|
|
45
72
|
/**
|
|
46
73
|
* This geometry gives the byte offset of every region inside one block. A
|
|
47
74
|
* block begins with one query scratch for each thread slot, and the slots
|
|
@@ -544,4 +571,4 @@ interface VectorWorkerError {
|
|
|
544
571
|
*/
|
|
545
572
|
type VectorWorkerMessage = VectorAckResponse | VectorInsertResponse | VectorSearchResponse | VectorOrdinalSearchResponse | VectorWorkerError;
|
|
546
573
|
|
|
547
|
-
export { ArenaSimd,
|
|
574
|
+
export type { ArenaSimd, BlockGeometry, GraphInsertOutcome, GrowableBuffer, HNSWSnapshot, OrdinalFilter, OsqBits, SharedFieldLoadRequest, SharedGraphHandles, SharedVectorFieldHandles, SharedVectorStoreHandles, VectorAckResponse, VectorBlockHandle, VectorBlockLayout, VectorBlockStorage, VectorCodeBlockLayout, VectorDropRequest, VectorInsertRequest, VectorInsertResponse, VectorLoadRequest, VectorMetric, VectorOrdinalSearchRequest, VectorOrdinalSearchResponse, VectorSearchRequest, VectorSearchResponse, VectorStoreSnapshot, VectorWorkerError, VectorWorkerMessage, VectorWorkerRequest, WorkerCopyCodes, WorkerCopySnapshot };
|
|
@@ -1,6 +1,10 @@
|
|
|
1
|
-
import { openSharedVectorField,
|
|
1
|
+
import { openSharedVectorField, createVectorStore } from '../chunk-UKSUENHK.mjs';
|
|
2
|
+
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-ULRPNYAG.mjs';
|
|
3
|
+
import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
|
|
4
|
+
import '../chunk-DJHHF2L5.mjs';
|
|
5
|
+
import { MAX_WORKER_COPIES } from '../chunk-Y2F66IQJ.mjs';
|
|
2
6
|
import '../chunk-ZJ4CSAB2.mjs';
|
|
3
|
-
import '../chunk-
|
|
7
|
+
import '../chunk-QZN3MIJ3.mjs';
|
|
4
8
|
import '../chunk-KEHK2O6P.mjs';
|
|
5
9
|
import '../chunk-R6A773WK.mjs';
|
|
6
10
|
import '../chunk-YTJCGKM4.mjs';
|
package/dist/workers/entry.mjs
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { startIdleHeapCleanup } from '../chunk-5XNIRLFC.mjs';
|
|
2
2
|
import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-YRX6XMOQ.mjs';
|
|
3
|
+
import { loadVectorCode } from '../chunk-R7OSDAFK.mjs';
|
|
4
|
+
import '../chunk-L5ORED7V.mjs';
|
|
3
5
|
import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-TPJU7NDJ.mjs';
|
|
4
6
|
|
|
5
7
|
// src/workers/action-handler.ts
|
|
@@ -18,7 +20,7 @@ async function importBootstrapModule(moduleUrl) {
|
|
|
18
20
|
await import(moduleUrl);
|
|
19
21
|
}
|
|
20
22
|
async function stopServingRequests() {
|
|
21
|
-
const { closeRequestThread } = await import('../thread-
|
|
23
|
+
const { closeRequestThread } = await import('../thread-GS65E6RM.mjs');
|
|
22
24
|
closeRequestThread();
|
|
23
25
|
}
|
|
24
26
|
function holdsCopies(executor) {
|
|
@@ -49,7 +51,7 @@ function createActionHandler(executor) {
|
|
|
49
51
|
if (!holdsCopies(executor)) {
|
|
50
52
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
|
|
51
53
|
}
|
|
52
|
-
const starting = import('../thread-
|
|
54
|
+
const starting = import('../thread-GS65E6RM.mjs').then(
|
|
53
55
|
({ serveRequests }) => serveRequests(executor, action.settings)
|
|
54
56
|
);
|
|
55
57
|
serving.start = starting.then(
|
|
@@ -102,7 +104,7 @@ async function setup() {
|
|
|
102
104
|
} catch {
|
|
103
105
|
parentPort = null;
|
|
104
106
|
}
|
|
105
|
-
const { createDirectExecutor } = await import('../direct-executor-
|
|
107
|
+
const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-LD6JW57R.mjs'), loadVectorCode()]);
|
|
106
108
|
const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
|
|
107
109
|
if (parentPort) {
|
|
108
110
|
const port = parentPort;
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-YCJLVNWR.mjs';
|
|
2
|
+
import './chunk-37CD3OKS.mjs';
|
|
3
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-IKST36MA.mjs';
|
|
4
|
+
import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-AQU2QHSY.mjs';
|
|
5
|
+
import { decodeStoredDocument } from './chunk-XBXKPWMR.mjs';
|
|
6
|
+
import './chunk-AXDAS2VA.mjs';
|
|
7
|
+
import './chunk-SYRFSF6V.mjs';
|
|
8
|
+
import './chunk-J2BPXTD7.mjs';
|
|
9
|
+
import './chunk-PR4O7CAP.mjs';
|
|
10
|
+
import './chunk-RVM7AODE.mjs';
|
|
11
|
+
import './chunk-AV4IJKD2.mjs';
|
|
12
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-Q5DC6WAB.mjs';
|
|
13
|
+
import './chunk-TGP5M5R7.mjs';
|
|
14
|
+
import './chunk-Y2F66IQJ.mjs';
|
|
15
|
+
import './chunk-ZJ4CSAB2.mjs';
|
|
16
|
+
import './chunk-QZN3MIJ3.mjs';
|
|
17
|
+
import './chunk-T6CNO72J.mjs';
|
|
18
|
+
import './chunk-WQBCBKY3.mjs';
|
|
19
|
+
import './chunk-PSUJCOSE.mjs';
|
|
20
|
+
import './chunk-CVQWQVFX.mjs';
|
|
21
|
+
import './chunk-FGMGUDHX.mjs';
|
|
22
|
+
import './chunk-KEHK2O6P.mjs';
|
|
23
|
+
import './chunk-GES7U2PM.mjs';
|
|
24
|
+
import './chunk-BVOJIYRV.mjs';
|
|
25
|
+
import './chunk-U2BKYEC7.mjs';
|
|
26
|
+
import './chunk-R6A773WK.mjs';
|
|
27
|
+
import './chunk-YTJCGKM4.mjs';
|
|
28
|
+
import './chunk-YRX6XMOQ.mjs';
|
|
29
|
+
import './chunk-R7OSDAFK.mjs';
|
|
30
|
+
import './chunk-L5ORED7V.mjs';
|
|
31
|
+
import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
|
|
32
|
+
import { decode } from '@msgpack/msgpack';
|
|
33
|
+
|
|
34
|
+
// src/engine/export/json-line.ts
|
|
35
|
+
function refuse(path, kind, docId) {
|
|
36
|
+
throw new NarsilError(
|
|
37
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
38
|
+
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
39
|
+
{ docId, field: path }
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
function encodeNumber(value, path, target) {
|
|
43
|
+
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
44
|
+
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
45
|
+
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
46
|
+
}
|
|
47
|
+
function encodeDate(value, path, target) {
|
|
48
|
+
const milliseconds = value.getTime();
|
|
49
|
+
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
50
|
+
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
51
|
+
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
52
|
+
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
53
|
+
}
|
|
54
|
+
function encodeBytes(value, path, target) {
|
|
55
|
+
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
56
|
+
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
57
|
+
}
|
|
58
|
+
function encodeList(value, path, target) {
|
|
59
|
+
target.parts.push("[");
|
|
60
|
+
for (let at = 0; at < value.length; at++) {
|
|
61
|
+
if (at > 0) target.parts.push(",");
|
|
62
|
+
encodeValue(value[at], `${path}[${at}]`, target);
|
|
63
|
+
}
|
|
64
|
+
target.parts.push("]");
|
|
65
|
+
}
|
|
66
|
+
function encodeMap(value, path, target) {
|
|
67
|
+
const proto = Object.getPrototypeOf(value);
|
|
68
|
+
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
69
|
+
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
70
|
+
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
71
|
+
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
72
|
+
for (let at = 0; at < entries.length; at++) {
|
|
73
|
+
const [key, entry] = entries[at];
|
|
74
|
+
if (at > 0) target.parts.push(",");
|
|
75
|
+
target.parts.push(JSON.stringify(key), ":");
|
|
76
|
+
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
77
|
+
}
|
|
78
|
+
target.parts.push(wrapped ? "}}" : "}");
|
|
79
|
+
}
|
|
80
|
+
function encodeValue(value, path, target) {
|
|
81
|
+
if (value === null) {
|
|
82
|
+
target.parts.push("null");
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
switch (typeof value) {
|
|
86
|
+
case "string":
|
|
87
|
+
target.parts.push(JSON.stringify(value));
|
|
88
|
+
return;
|
|
89
|
+
case "boolean":
|
|
90
|
+
target.parts.push(value ? "true" : "false");
|
|
91
|
+
return;
|
|
92
|
+
case "number":
|
|
93
|
+
target.parts.push(encodeNumber(value, path, target));
|
|
94
|
+
return;
|
|
95
|
+
case "object":
|
|
96
|
+
break;
|
|
97
|
+
default:
|
|
98
|
+
refuse(path, `a ${typeof value}`, target.docId);
|
|
99
|
+
}
|
|
100
|
+
if (Array.isArray(value)) encodeList(value, path, target);
|
|
101
|
+
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
102
|
+
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
103
|
+
else encodeMap(value, path, target);
|
|
104
|
+
}
|
|
105
|
+
function encodeExactJson(value, docId) {
|
|
106
|
+
const target = { docId, tagged: false, parts: [] };
|
|
107
|
+
encodeValue(value, "", target);
|
|
108
|
+
return target.parts.join("");
|
|
109
|
+
}
|
|
110
|
+
function encodeDocumentJson(document, docId) {
|
|
111
|
+
const target = { docId, tagged: true, parts: [] };
|
|
112
|
+
encodeValue(document, "", target);
|
|
113
|
+
return target.parts.join("");
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// src/engine/export/write.ts
|
|
117
|
+
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
118
|
+
var FLOAT32_BYTES = 4;
|
|
119
|
+
async function locateVectors(checkpoint) {
|
|
120
|
+
const located = /* @__PURE__ */ new Map();
|
|
121
|
+
for (const field of checkpoint.manifest.vectors) {
|
|
122
|
+
const byDocId = /* @__PURE__ */ new Map();
|
|
123
|
+
for (const ref of field.files) {
|
|
124
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
125
|
+
envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
|
|
126
|
+
what: `vector file "${ref.key}"`,
|
|
127
|
+
allowsCompression: false
|
|
128
|
+
});
|
|
129
|
+
const decoded = decodeVectorFilePayload(decode(payload));
|
|
130
|
+
for (let position = 0; position < decoded.docIds.length; position++) {
|
|
131
|
+
if (isDead(ref.dead, position)) continue;
|
|
132
|
+
byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
located.set(field.fieldPath, byDocId);
|
|
136
|
+
}
|
|
137
|
+
return located;
|
|
138
|
+
}
|
|
139
|
+
async function readVector(checkpoint, location) {
|
|
140
|
+
const width = location.dimension * FLOAT32_BYTES;
|
|
141
|
+
const offset = location.ref.bytes - (location.ref.count - location.position) * width;
|
|
142
|
+
return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
|
|
143
|
+
}
|
|
144
|
+
async function readSegment(checkpoint, ref) {
|
|
145
|
+
const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
|
|
146
|
+
envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
|
|
147
|
+
what: `segment "${ref.key}"`,
|
|
148
|
+
allowsCompression: true
|
|
149
|
+
});
|
|
150
|
+
return decodeKeywordSegment(payload, false);
|
|
151
|
+
}
|
|
152
|
+
var decoder = new TextDecoder();
|
|
153
|
+
function idAt(contents, ordinal) {
|
|
154
|
+
const ids = contents.arrays.idTable;
|
|
155
|
+
return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
|
|
156
|
+
}
|
|
157
|
+
async function winnersOf(checkpoint, partition) {
|
|
158
|
+
const winners = /* @__PURE__ */ new Map();
|
|
159
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
160
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
161
|
+
for (const docId of contents.tombstones) winners.delete(docId);
|
|
162
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
|
|
163
|
+
}
|
|
164
|
+
return winners;
|
|
165
|
+
}
|
|
166
|
+
async function* documentLines(checkpoint) {
|
|
167
|
+
const vectors = await locateVectors(checkpoint);
|
|
168
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
169
|
+
const winners = await winnersOf(checkpoint, partition);
|
|
170
|
+
for (let at = 0; at < partition.segments.length; at++) {
|
|
171
|
+
const contents = await readSegment(checkpoint, partition.segments[at]);
|
|
172
|
+
const table = contents.arrays.documentTable;
|
|
173
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
|
|
174
|
+
const docId = idAt(contents, ordinal);
|
|
175
|
+
if (winners.get(docId) !== at) continue;
|
|
176
|
+
const stored = decodeStoredDocument(
|
|
177
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
178
|
+
table.discardedFields
|
|
179
|
+
);
|
|
180
|
+
const document = { id: docId, ...stored };
|
|
181
|
+
document.id = docId;
|
|
182
|
+
for (const [fieldPath, located] of vectors) {
|
|
183
|
+
const location = located.get(docId);
|
|
184
|
+
if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
|
|
185
|
+
}
|
|
186
|
+
yield `${encodeDocumentJson(document, docId)}
|
|
187
|
+
`;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
async function* exportLines(header, checkpoint) {
|
|
193
|
+
yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
|
|
194
|
+
`;
|
|
195
|
+
yield* documentLines(checkpoint);
|
|
196
|
+
}
|
|
197
|
+
function documentExportStream(header, checkpoint) {
|
|
198
|
+
const lines = exportLines(header, checkpoint);
|
|
199
|
+
const encoder = new TextEncoder();
|
|
200
|
+
let released = false;
|
|
201
|
+
const release = () => {
|
|
202
|
+
if (released) return;
|
|
203
|
+
released = true;
|
|
204
|
+
checkpoint.release();
|
|
205
|
+
};
|
|
206
|
+
return new ReadableStream({
|
|
207
|
+
async pull(controller) {
|
|
208
|
+
try {
|
|
209
|
+
let text = "";
|
|
210
|
+
while (text.length < EXPORT_CHUNK_BYTES) {
|
|
211
|
+
const step = await lines.next();
|
|
212
|
+
if (step.done) {
|
|
213
|
+
if (text.length > 0) controller.enqueue(encoder.encode(text));
|
|
214
|
+
release();
|
|
215
|
+
controller.close();
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
text += step.value;
|
|
219
|
+
}
|
|
220
|
+
controller.enqueue(encoder.encode(text));
|
|
221
|
+
} catch (error) {
|
|
222
|
+
release();
|
|
223
|
+
controller.error(error);
|
|
224
|
+
}
|
|
225
|
+
},
|
|
226
|
+
async cancel() {
|
|
227
|
+
release();
|
|
228
|
+
await lines.return(void 0);
|
|
229
|
+
}
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export { documentExportStream };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@delali/narsil",
|
|
3
|
-
"version": "0.3.1-next.
|
|
3
|
+
"version": "0.3.1-next.87",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
|
|
6
6
|
"author": "Delali (https://sondelali.com)",
|
|
@@ -51,6 +51,10 @@
|
|
|
51
51
|
"browser": "./src/vector/native/loader.browser.ts",
|
|
52
52
|
"default": "./src/vector/native/loader.ts"
|
|
53
53
|
},
|
|
54
|
+
"#platform/search-backend-settings": {
|
|
55
|
+
"browser": "./src/engine/search-backend-settings.browser.ts",
|
|
56
|
+
"default": "./src/engine/search-backend-settings.ts"
|
|
57
|
+
},
|
|
54
58
|
"#platform/native-crc32": {
|
|
55
59
|
"browser": "./src/serialization/native-crc32.browser.ts",
|
|
56
60
|
"default": "./src/serialization/native-crc32.ts"
|
|
@@ -246,11 +250,11 @@
|
|
|
246
250
|
}
|
|
247
251
|
},
|
|
248
252
|
"optionalDependencies": {
|
|
249
|
-
"@delali/narsil-native-darwin-arm64": "0.3.1-next.
|
|
250
|
-
"@delali/narsil-native-darwin-x64": "0.3.1-next.
|
|
251
|
-
"@delali/narsil-native-linux-arm64": "0.3.1-next.
|
|
252
|
-
"@delali/narsil-native-linux-x64": "0.3.1-next.
|
|
253
|
-
"@delali/narsil-native-win32-arm64": "0.3.1-next.
|
|
254
|
-
"@delali/narsil-native-win32-x64": "0.3.1-next.
|
|
253
|
+
"@delali/narsil-native-darwin-arm64": "0.3.1-next.87",
|
|
254
|
+
"@delali/narsil-native-darwin-x64": "0.3.1-next.87",
|
|
255
|
+
"@delali/narsil-native-linux-arm64": "0.3.1-next.87",
|
|
256
|
+
"@delali/narsil-native-linux-x64": "0.3.1-next.87",
|
|
257
|
+
"@delali/narsil-native-win32-arm64": "0.3.1-next.87",
|
|
258
|
+
"@delali/narsil-native-win32-x64": "0.3.1-next.87"
|
|
255
259
|
}
|
|
256
260
|
}
|