@delali/narsil 0.3.1-next.85 → 0.3.1-next.87

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/dist/browser/chunk-3BXQPIXK.mjs +602 -0
  2. package/dist/browser/chunk-3ZRMZIH4.mjs +3959 -0
  3. package/dist/browser/chunk-BBDAYUCO.mjs +2703 -0
  4. package/dist/browser/chunk-FHFFWLEK.mjs +669 -0
  5. package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
  6. package/dist/browser/chunk-NQW2YXEX.mjs +346 -0
  7. package/dist/browser/chunk-TBMNPGFO.mjs +22910 -0
  8. package/dist/browser/durability-wiring-7RUN7LNS.mjs +3250 -0
  9. package/dist/browser/module-E25UDADM.mjs +2766 -0
  10. package/dist/browser/snapshot-A42DRERF.mjs +888 -0
  11. package/dist/browser/write-4U6Q26TJ.mjs +206 -0
  12. package/dist/{capabilities-CAdkp2P-.d.ts → capabilities-SJPttcgQ.d.ts} +2 -2
  13. package/dist/chunk-37CD3OKS.mjs +87 -0
  14. package/dist/chunk-AQU2QHSY.mjs +7247 -0
  15. package/dist/{chunk-OLXM65YH.mjs → chunk-AV4IJKD2.mjs} +1 -18
  16. package/dist/{chunk-4D6OH4EF.mjs → chunk-AXDAS2VA.mjs} +7 -5
  17. package/dist/{chunk-LS5IRM23.mjs → chunk-BVOJIYRV.mjs} +2 -25
  18. package/dist/chunk-C7SLA6CM.mjs +19 -0
  19. package/dist/{chunk-KJ7LB3TR.mjs → chunk-CVQWQVFX.mjs} +3 -22
  20. package/dist/chunk-DJHHF2L5.mjs +17 -0
  21. package/dist/chunk-FGMGUDHX.mjs +21 -0
  22. package/dist/chunk-IKST36MA.mjs +614 -0
  23. package/dist/{chunk-2453RJYK.mjs → chunk-J2BPXTD7.mjs} +167 -176
  24. package/dist/chunk-JFJBA3PD.mjs +522 -0
  25. package/dist/chunk-K4OWQ2M3.mjs +462 -0
  26. package/dist/chunk-L5ORED7V.mjs +25 -0
  27. package/dist/chunk-MKK4OZ5W.mjs +703 -0
  28. package/dist/chunk-MXS5HWIH.mjs +515 -0
  29. package/dist/chunk-P36MUFEJ.mjs +10 -0
  30. package/dist/chunk-PR4O7CAP.mjs +18 -0
  31. package/dist/{chunk-4VBG6PEH.mjs → chunk-PSUJCOSE.mjs} +3 -2
  32. package/dist/{chunk-ZCKPRCYI.mjs → chunk-Q5DC6WAB.mjs} +1 -1
  33. package/dist/{chunk-NFF5TDRG.mjs → chunk-QZN3MIJ3.mjs} +1 -16
  34. package/dist/chunk-R7OSDAFK.mjs +44 -0
  35. package/dist/chunk-RVM7AODE.mjs +73 -0
  36. package/dist/{chunk-XYDP7CCJ.mjs → chunk-SYRFSF6V.mjs} +6 -4
  37. package/dist/{chunk-SREPGC6O.mjs → chunk-T6CNO72J.mjs} +1 -1
  38. package/dist/chunk-T7DMUVTL.mjs +1714 -0
  39. package/dist/chunk-UKSUENHK.mjs +997 -0
  40. package/dist/{chunk-NIWDQTZB.mjs → chunk-ULRPNYAG.mjs} +1220 -2311
  41. package/dist/{chunk-GNYXCLJS.mjs → chunk-URGLSNKR.mjs} +4 -3
  42. package/dist/chunk-WQSHDZ7Z.mjs +631 -0
  43. package/dist/{chunk-RCSXJCI5.mjs → chunk-XBXKPWMR.mjs} +20 -2664
  44. package/dist/{chunk-RM6FRYEC.mjs → chunk-XSZVS4S3.mjs} +3 -2
  45. package/dist/chunk-Y2F66IQJ.mjs +74 -0
  46. package/dist/chunk-YCJLVNWR.mjs +262 -0
  47. package/dist/{client-CKUtkLz3.d.ts → client-BMK53IR2.d.ts} +3 -3
  48. package/dist/client.d.ts +8 -8
  49. package/dist/{config-CSDEPraV.d.ts → config-BUR-7eVQ.d.ts} +2 -2
  50. package/dist/{config-BuIkeM5C.d.ts → config-UJx1dyX1.d.ts} +2 -2
  51. package/dist/direct-executor-LD6JW57R.mjs +18 -0
  52. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  53. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  54. package/dist/distribution.d.ts +8 -8
  55. package/dist/distribution.mjs +30 -15
  56. package/dist/durability-wiring-5S32KDDC.mjs +2797 -0
  57. package/dist/{engine-DMNZJAUq.d.ts → engine-9X6W5naj.d.ts} +2 -2
  58. package/dist/{index-DzONZUKE.d.ts → index-vCMu8OeQ.d.ts} +1 -1
  59. package/dist/index.browser.mjs +2 -38121
  60. package/dist/index.d.ts +23 -9
  61. package/dist/index.mjs +18 -14
  62. package/dist/{memory-BPKNx9Tl.d.ts → memory-CV9NAxLY.d.ts} +1 -1
  63. package/dist/module-WIDYIJLY.mjs +1819 -0
  64. package/dist/partial-words.mjs +5 -3
  65. package/dist/pattern.mjs +3 -2
  66. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  67. package/dist/persistence/durability/checkpoint-worker.mjs +23 -13
  68. package/dist/react.d.ts +7 -7
  69. package/dist/{schema-DgyDijhf.d.ts → schema-DtsgmNfU.d.ts} +1 -1
  70. package/dist/server.d.ts +10 -10
  71. package/dist/server.mjs +14 -9
  72. package/dist/snapshot-6BYSUPWL.mjs +469 -0
  73. package/dist/{tasks-DGyz586z.d.ts → tasks-BGaZJyoY.d.ts} +1 -1
  74. package/dist/{thread-EUUNJNKU.mjs → thread-GS65E6RM.mjs} +12 -9
  75. package/dist/vector/search-worker.d.ts +30 -3
  76. package/dist/vector/search-worker.mjs +6 -2
  77. package/dist/workers/entry.mjs +5 -3
  78. package/dist/write-HLEZTL2L.mjs +233 -0
  79. package/package.json +11 -7
  80. package/dist/chunk-67MQF5CV.mjs +0 -15198
  81. package/dist/direct-executor-NY7ZTZIH.mjs +0 -19
  82. package/dist/simd-DW5AtJ6B.d.ts +0 -41
@@ -1,18 +1,21 @@
1
- import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-RM6FRYEC.mjs';
2
- import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-4D6OH4EF.mjs';
3
- import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-XYDP7CCJ.mjs';
4
- import './chunk-ZCKPRCYI.mjs';
1
+ import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery, refuseRegex, awaitDepartureConfirmation, departureConfirmation, REQUEST_THREAD_DEPARTURE_DRAIN_MS, departureConfirmed, encodeJson } from './chunk-XSZVS4S3.mjs';
2
+ import './chunk-DJHHF2L5.mjs';
3
+ import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-AXDAS2VA.mjs';
4
+ import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-SYRFSF6V.mjs';
5
+ import './chunk-Q5DC6WAB.mjs';
5
6
  import './chunk-ZJ4CSAB2.mjs';
6
- import './chunk-NFF5TDRG.mjs';
7
- import './chunk-SREPGC6O.mjs';
7
+ import './chunk-QZN3MIJ3.mjs';
8
+ import './chunk-T6CNO72J.mjs';
8
9
  import './chunk-WQBCBKY3.mjs';
9
- import './chunk-4VBG6PEH.mjs';
10
- import { extractVectorFieldsFromSchema } from './chunk-KJ7LB3TR.mjs';
10
+ import './chunk-PSUJCOSE.mjs';
11
+ import { extractVectorFieldsFromSchema } from './chunk-CVQWQVFX.mjs';
12
+ import './chunk-FGMGUDHX.mjs';
11
13
  import './chunk-KEHK2O6P.mjs';
12
14
  import './chunk-GES7U2PM.mjs';
13
- import './chunk-LS5IRM23.mjs';
15
+ import './chunk-BVOJIYRV.mjs';
14
16
  import './chunk-R6A773WK.mjs';
15
17
  import './chunk-YTJCGKM4.mjs';
18
+ import './chunk-L5ORED7V.mjs';
16
19
  import { ServerErrorCodes, ErrorCodes, NarsilError } from './chunk-TPJU7NDJ.mjs';
17
20
 
18
21
  // src/server/request-threads/held-vectors.ts
@@ -1,5 +1,4 @@
1
- import { e as VectorQuantizationMode } from '../schema-DgyDijhf.js';
2
- import { A as ArenaSimd } from '../simd-DW5AtJ6B.js';
1
+ import { e as VectorQuantizationMode } from '../schema-DtsgmNfU.js';
3
2
  import '../adapters-79OIa4xm.js';
4
3
 
5
4
  /**
@@ -42,6 +41,34 @@ interface OrdinalFilter {
42
41
  */
43
42
  type GrowableBuffer = SharedArrayBuffer | ArrayBuffer;
44
43
 
44
+ /**
45
+ * The distance kernels bound to one memory, which read their operands at byte
46
+ * offsets inside that memory so that a caller measures against a vector where
47
+ * it already lies.
48
+ *
49
+ * @internal
50
+ */
51
+ interface ArenaSimd {
52
+ /** The kernels read every operand out of this memory. */
53
+ memory: WebAssembly.Memory;
54
+ /** Returns the dot product of two float32 vectors at the given byte offsets. */
55
+ dot_product: (ptrA: number, ptrB: number, len: number) => number;
56
+ /** Returns the length of the float32 vector at the given byte offset. */
57
+ magnitude: (ptr: number, len: number) => number;
58
+ /** Returns the squared euclidean distance between two float32 vectors at the given byte offsets. */
59
+ squared_euclidean_distance: (ptrA: number, ptrB: number, len: number) => number;
60
+ /** Returns the integer dot product of two byte code vectors at the given byte offsets. */
61
+ dot_u8: (ptrA: number, ptrB: number, len: number) => number;
62
+ /** Returns the integer squared distance between two byte code vectors at the given byte offsets. */
63
+ sqdist_u8: (ptrA: number, ptrB: number, len: number) => number;
64
+ /** Returns the sum of level products of two 4-bit codes at the given byte offsets, each holding two levels per byte. */
65
+ osq_dot_nibbles_4x4: (ptrA: number, ptrB: number, bytes: number) => number;
66
+ /** Returns the sum of level products of a 1-bit or 2-bit code and a query staged as four bit planes. Each plane spans `planeBytes`, with one bit for each of the code's lowest level bits. */
67
+ osq_dot_bits: (ptrDoc: number, ptrPlanes: number, planeBytes: number, docBits: number) => number;
68
+ /** Returns the sum of level products of two 1-bit codes or two 2-bit codes at the given byte offsets. */
69
+ osq_dot_bits_pair: (ptrA: number, ptrB: number, bytes: number, bits: number) => number;
70
+ }
71
+
45
72
  /**
46
73
  * This geometry gives the byte offset of every region inside one block. A
47
74
  * block begins with one query scratch for each thread slot, and the slots
@@ -544,4 +571,4 @@ interface VectorWorkerError {
544
571
  */
545
572
  type VectorWorkerMessage = VectorAckResponse | VectorInsertResponse | VectorSearchResponse | VectorOrdinalSearchResponse | VectorWorkerError;
546
573
 
547
- export { ArenaSimd, type BlockGeometry, type GraphInsertOutcome, type GrowableBuffer, type HNSWSnapshot, type OrdinalFilter, type OsqBits, type SharedFieldLoadRequest, type SharedGraphHandles, type SharedVectorFieldHandles, type SharedVectorStoreHandles, type VectorAckResponse, type VectorBlockHandle, type VectorBlockLayout, type VectorBlockStorage, type VectorCodeBlockLayout, type VectorDropRequest, type VectorInsertRequest, type VectorInsertResponse, type VectorLoadRequest, type VectorMetric, type VectorOrdinalSearchRequest, type VectorOrdinalSearchResponse, type VectorSearchRequest, type VectorSearchResponse, type VectorStoreSnapshot, type VectorWorkerError, type VectorWorkerMessage, type VectorWorkerRequest, type WorkerCopyCodes, type WorkerCopySnapshot };
574
+ export type { ArenaSimd, BlockGeometry, GraphInsertOutcome, GrowableBuffer, HNSWSnapshot, OrdinalFilter, OsqBits, SharedFieldLoadRequest, SharedGraphHandles, SharedVectorFieldHandles, SharedVectorStoreHandles, VectorAckResponse, VectorBlockHandle, VectorBlockLayout, VectorBlockStorage, VectorCodeBlockLayout, VectorDropRequest, VectorInsertRequest, VectorInsertResponse, VectorLoadRequest, VectorMetric, VectorOrdinalSearchRequest, VectorOrdinalSearchResponse, VectorSearchRequest, VectorSearchResponse, VectorStoreSnapshot, VectorWorkerError, VectorWorkerMessage, VectorWorkerRequest, WorkerCopyCodes, WorkerCopySnapshot };
@@ -1,6 +1,10 @@
1
- import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-NIWDQTZB.mjs';
1
+ import { openSharedVectorField, createVectorStore } from '../chunk-UKSUENHK.mjs';
2
+ import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-ULRPNYAG.mjs';
3
+ import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
4
+ import '../chunk-DJHHF2L5.mjs';
5
+ import { MAX_WORKER_COPIES } from '../chunk-Y2F66IQJ.mjs';
2
6
  import '../chunk-ZJ4CSAB2.mjs';
3
- import '../chunk-NFF5TDRG.mjs';
7
+ import '../chunk-QZN3MIJ3.mjs';
4
8
  import '../chunk-KEHK2O6P.mjs';
5
9
  import '../chunk-R6A773WK.mjs';
6
10
  import '../chunk-YTJCGKM4.mjs';
@@ -1,5 +1,7 @@
1
1
  import { startIdleHeapCleanup } from '../chunk-5XNIRLFC.mjs';
2
2
  import { threadSlotOfWorker, isValidWorkerAction } from '../chunk-YRX6XMOQ.mjs';
3
+ import { loadVectorCode } from '../chunk-R7OSDAFK.mjs';
4
+ import '../chunk-L5ORED7V.mjs';
3
5
  import { ErrorCodes, NarsilError, isNarsilError } from '../chunk-TPJU7NDJ.mjs';
4
6
 
5
7
  // src/workers/action-handler.ts
@@ -18,7 +20,7 @@ async function importBootstrapModule(moduleUrl) {
18
20
  await import(moduleUrl);
19
21
  }
20
22
  async function stopServingRequests() {
21
- const { closeRequestThread } = await import('../thread-EUUNJNKU.mjs');
23
+ const { closeRequestThread } = await import('../thread-GS65E6RM.mjs');
22
24
  closeRequestThread();
23
25
  }
24
26
  function holdsCopies(executor) {
@@ -49,7 +51,7 @@ function createActionHandler(executor) {
49
51
  if (!holdsCopies(executor)) {
50
52
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
51
53
  }
52
- const starting = import('../thread-EUUNJNKU.mjs').then(
54
+ const starting = import('../thread-GS65E6RM.mjs').then(
53
55
  ({ serveRequests }) => serveRequests(executor, action.settings)
54
56
  );
55
57
  serving.start = starting.then(
@@ -102,7 +104,7 @@ async function setup() {
102
104
  } catch {
103
105
  parentPort = null;
104
106
  }
105
- const { createDirectExecutor } = await import('../direct-executor-NY7ZTZIH.mjs');
107
+ const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-LD6JW57R.mjs'), loadVectorCode()]);
106
108
  const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
107
109
  if (parentPort) {
108
110
  const port = parentPort;
@@ -0,0 +1,233 @@
1
+ import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-YCJLVNWR.mjs';
2
+ import './chunk-37CD3OKS.mjs';
3
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-IKST36MA.mjs';
4
+ import { openChecksummedEnvelope, NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-AQU2QHSY.mjs';
5
+ import { decodeStoredDocument } from './chunk-XBXKPWMR.mjs';
6
+ import './chunk-AXDAS2VA.mjs';
7
+ import './chunk-SYRFSF6V.mjs';
8
+ import './chunk-J2BPXTD7.mjs';
9
+ import './chunk-PR4O7CAP.mjs';
10
+ import './chunk-RVM7AODE.mjs';
11
+ import './chunk-AV4IJKD2.mjs';
12
+ import { EXPORT_CHUNK_BYTES } from './chunk-Q5DC6WAB.mjs';
13
+ import './chunk-TGP5M5R7.mjs';
14
+ import './chunk-Y2F66IQJ.mjs';
15
+ import './chunk-ZJ4CSAB2.mjs';
16
+ import './chunk-QZN3MIJ3.mjs';
17
+ import './chunk-T6CNO72J.mjs';
18
+ import './chunk-WQBCBKY3.mjs';
19
+ import './chunk-PSUJCOSE.mjs';
20
+ import './chunk-CVQWQVFX.mjs';
21
+ import './chunk-FGMGUDHX.mjs';
22
+ import './chunk-KEHK2O6P.mjs';
23
+ import './chunk-GES7U2PM.mjs';
24
+ import './chunk-BVOJIYRV.mjs';
25
+ import './chunk-U2BKYEC7.mjs';
26
+ import './chunk-R6A773WK.mjs';
27
+ import './chunk-YTJCGKM4.mjs';
28
+ import './chunk-YRX6XMOQ.mjs';
29
+ import './chunk-R7OSDAFK.mjs';
30
+ import './chunk-L5ORED7V.mjs';
31
+ import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
32
+ import { decode } from '@msgpack/msgpack';
33
+
34
+ // src/engine/export/json-line.ts
35
+ function refuse(path, kind, docId) {
36
+ throw new NarsilError(
37
+ ErrorCodes.DOC_VALIDATION_FAILED,
38
+ `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
39
+ { docId, field: path }
40
+ );
41
+ }
42
+ function encodeNumber(value, path, target) {
43
+ if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
44
+ if (!target.tagged) refuse(path, String(value), target.docId);
45
+ return `{"${NUMBER_TAG}":"${String(value)}"}`;
46
+ }
47
+ function encodeDate(value, path, target) {
48
+ const milliseconds = value.getTime();
49
+ if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
50
+ const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
51
+ const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
52
+ return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
53
+ }
54
+ function encodeBytes(value, path, target) {
55
+ if (!target.tagged) refuse(path, "bytes", target.docId);
56
+ return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
57
+ }
58
+ function encodeList(value, path, target) {
59
+ target.parts.push("[");
60
+ for (let at = 0; at < value.length; at++) {
61
+ if (at > 0) target.parts.push(",");
62
+ encodeValue(value[at], `${path}[${at}]`, target);
63
+ }
64
+ target.parts.push("]");
65
+ }
66
+ function encodeMap(value, path, target) {
67
+ const proto = Object.getPrototypeOf(value);
68
+ if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
69
+ const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
70
+ const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
71
+ target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
72
+ for (let at = 0; at < entries.length; at++) {
73
+ const [key, entry] = entries[at];
74
+ if (at > 0) target.parts.push(",");
75
+ target.parts.push(JSON.stringify(key), ":");
76
+ encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
77
+ }
78
+ target.parts.push(wrapped ? "}}" : "}");
79
+ }
80
+ function encodeValue(value, path, target) {
81
+ if (value === null) {
82
+ target.parts.push("null");
83
+ return;
84
+ }
85
+ switch (typeof value) {
86
+ case "string":
87
+ target.parts.push(JSON.stringify(value));
88
+ return;
89
+ case "boolean":
90
+ target.parts.push(value ? "true" : "false");
91
+ return;
92
+ case "number":
93
+ target.parts.push(encodeNumber(value, path, target));
94
+ return;
95
+ case "object":
96
+ break;
97
+ default:
98
+ refuse(path, `a ${typeof value}`, target.docId);
99
+ }
100
+ if (Array.isArray(value)) encodeList(value, path, target);
101
+ else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
102
+ else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
103
+ else encodeMap(value, path, target);
104
+ }
105
+ function encodeExactJson(value, docId) {
106
+ const target = { docId, tagged: false, parts: [] };
107
+ encodeValue(value, "", target);
108
+ return target.parts.join("");
109
+ }
110
+ function encodeDocumentJson(document, docId) {
111
+ const target = { docId, tagged: true, parts: [] };
112
+ encodeValue(document, "", target);
113
+ return target.parts.join("");
114
+ }
115
+
116
+ // src/engine/export/write.ts
117
+ var VECTOR_FILE_ENVELOPE_VERSION = 2;
118
+ var FLOAT32_BYTES = 4;
119
+ async function locateVectors(checkpoint) {
120
+ const located = /* @__PURE__ */ new Map();
121
+ for (const field of checkpoint.manifest.vectors) {
122
+ const byDocId = /* @__PURE__ */ new Map();
123
+ for (const ref of field.files) {
124
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
125
+ envelopeFormatVersion: VECTOR_FILE_ENVELOPE_VERSION,
126
+ what: `vector file "${ref.key}"`,
127
+ allowsCompression: false
128
+ });
129
+ const decoded = decodeVectorFilePayload(decode(payload));
130
+ for (let position = 0; position < decoded.docIds.length; position++) {
131
+ if (isDead(ref.dead, position)) continue;
132
+ byDocId.set(decoded.docIds[position], { ref, position, dimension: decoded.dimension });
133
+ }
134
+ }
135
+ located.set(field.fieldPath, byDocId);
136
+ }
137
+ return located;
138
+ }
139
+ async function readVector(checkpoint, location) {
140
+ const width = location.dimension * FLOAT32_BYTES;
141
+ const offset = location.ref.bytes - (location.ref.count - location.position) * width;
142
+ return Array.from(bytesToVectors((await checkpoint.readRange(location.ref, offset, width)).slice()));
143
+ }
144
+ async function readSegment(checkpoint, ref) {
145
+ const { payload } = await openChecksummedEnvelope(await checkpoint.readFile(ref), {
146
+ envelopeFormatVersion: KEYWORD_SEGMENT_ENVELOPE_VERSION,
147
+ what: `segment "${ref.key}"`,
148
+ allowsCompression: true
149
+ });
150
+ return decodeKeywordSegment(payload, false);
151
+ }
152
+ var decoder = new TextDecoder();
153
+ function idAt(contents, ordinal) {
154
+ const ids = contents.arrays.idTable;
155
+ return decoder.decode(ids.blob.subarray(ids.offsets[ordinal], ids.offsets[ordinal + 1]));
156
+ }
157
+ async function winnersOf(checkpoint, partition) {
158
+ const winners = /* @__PURE__ */ new Map();
159
+ for (let at = 0; at < partition.segments.length; at++) {
160
+ const contents = await readSegment(checkpoint, partition.segments[at]);
161
+ for (const docId of contents.tombstones) winners.delete(docId);
162
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) winners.set(idAt(contents, ordinal), at);
163
+ }
164
+ return winners;
165
+ }
166
+ async function* documentLines(checkpoint) {
167
+ const vectors = await locateVectors(checkpoint);
168
+ for (const partition of checkpoint.manifest.partitions) {
169
+ const winners = await winnersOf(checkpoint, partition);
170
+ for (let at = 0; at < partition.segments.length; at++) {
171
+ const contents = await readSegment(checkpoint, partition.segments[at]);
172
+ const table = contents.arrays.documentTable;
173
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) {
174
+ const docId = idAt(contents, ordinal);
175
+ if (winners.get(docId) !== at) continue;
176
+ const stored = decodeStoredDocument(
177
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
178
+ table.discardedFields
179
+ );
180
+ const document = { id: docId, ...stored };
181
+ document.id = docId;
182
+ for (const [fieldPath, located] of vectors) {
183
+ const location = located.get(docId);
184
+ if (location !== void 0) setNestedValue(document, fieldPath, await readVector(checkpoint, location));
185
+ }
186
+ yield `${encodeDocumentJson(document, docId)}
187
+ `;
188
+ }
189
+ }
190
+ }
191
+ }
192
+ async function* exportLines(header, checkpoint) {
193
+ yield `${encodeExactJson({ format: DOCUMENT_EXPORT_FORMAT, version: DOCUMENT_EXPORT_VERSION, index: header }, "")}
194
+ `;
195
+ yield* documentLines(checkpoint);
196
+ }
197
+ function documentExportStream(header, checkpoint) {
198
+ const lines = exportLines(header, checkpoint);
199
+ const encoder = new TextEncoder();
200
+ let released = false;
201
+ const release = () => {
202
+ if (released) return;
203
+ released = true;
204
+ checkpoint.release();
205
+ };
206
+ return new ReadableStream({
207
+ async pull(controller) {
208
+ try {
209
+ let text = "";
210
+ while (text.length < EXPORT_CHUNK_BYTES) {
211
+ const step = await lines.next();
212
+ if (step.done) {
213
+ if (text.length > 0) controller.enqueue(encoder.encode(text));
214
+ release();
215
+ controller.close();
216
+ return;
217
+ }
218
+ text += step.value;
219
+ }
220
+ controller.enqueue(encoder.encode(text));
221
+ } catch (error) {
222
+ release();
223
+ controller.error(error);
224
+ }
225
+ },
226
+ async cancel() {
227
+ release();
228
+ await lines.return(void 0);
229
+ }
230
+ });
231
+ }
232
+
233
+ export { documentExportStream };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@delali/narsil",
3
- "version": "0.3.1-next.85",
3
+ "version": "0.3.1-next.87",
4
4
  "type": "module",
5
5
  "description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
6
6
  "author": "Delali (https://sondelali.com)",
@@ -51,6 +51,10 @@
51
51
  "browser": "./src/vector/native/loader.browser.ts",
52
52
  "default": "./src/vector/native/loader.ts"
53
53
  },
54
+ "#platform/search-backend-settings": {
55
+ "browser": "./src/engine/search-backend-settings.browser.ts",
56
+ "default": "./src/engine/search-backend-settings.ts"
57
+ },
54
58
  "#platform/native-crc32": {
55
59
  "browser": "./src/serialization/native-crc32.browser.ts",
56
60
  "default": "./src/serialization/native-crc32.ts"
@@ -246,11 +250,11 @@
246
250
  }
247
251
  },
248
252
  "optionalDependencies": {
249
- "@delali/narsil-native-darwin-arm64": "0.3.1-next.85",
250
- "@delali/narsil-native-darwin-x64": "0.3.1-next.85",
251
- "@delali/narsil-native-linux-arm64": "0.3.1-next.85",
252
- "@delali/narsil-native-linux-x64": "0.3.1-next.85",
253
- "@delali/narsil-native-win32-arm64": "0.3.1-next.85",
254
- "@delali/narsil-native-win32-x64": "0.3.1-next.85"
253
+ "@delali/narsil-native-darwin-arm64": "0.3.1-next.87",
254
+ "@delali/narsil-native-darwin-x64": "0.3.1-next.87",
255
+ "@delali/narsil-native-linux-arm64": "0.3.1-next.87",
256
+ "@delali/narsil-native-linux-x64": "0.3.1-next.87",
257
+ "@delali/narsil-native-win32-arm64": "0.3.1-next.87",
258
+ "@delali/narsil-native-win32-x64": "0.3.1-next.87"
255
259
  }
256
260
  }