@delali/narsil 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -936,96 +936,6 @@ function heapPop(heap) {
|
|
|
936
936
|
}
|
|
937
937
|
return top;
|
|
938
938
|
}
|
|
939
|
-
function encodeRawPayloadV2(wire) {
|
|
940
|
-
return encode(wire);
|
|
941
|
-
}
|
|
942
|
-
var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
|
|
943
|
-
function validateHnswMetric(value) {
|
|
944
|
-
if (value === void 0 || value === null) return void 0;
|
|
945
|
-
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
946
|
-
return value;
|
|
947
|
-
}
|
|
948
|
-
return void 0;
|
|
949
|
-
}
|
|
950
|
-
function deserializePayloadV2(data) {
|
|
951
|
-
const raw = decode(data);
|
|
952
|
-
const documents = {};
|
|
953
|
-
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
954
|
-
documents[docId] = {
|
|
955
|
-
fields: doc.fields,
|
|
956
|
-
fieldLengths: doc.field_lengths ?? {}
|
|
957
|
-
};
|
|
958
|
-
}
|
|
959
|
-
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
960
|
-
const invertedIndex = {};
|
|
961
|
-
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
962
|
-
const count = list.ids.length;
|
|
963
|
-
const postings = new Array(count);
|
|
964
|
-
for (let i = 0; i < count; i++) {
|
|
965
|
-
postings[i] = {
|
|
966
|
-
docId: list.ids[i],
|
|
967
|
-
termFrequency: list.tf[i],
|
|
968
|
-
field: fieldNames[list.fi[i]] ?? "",
|
|
969
|
-
positions: list.pos ? list.pos[i] : []
|
|
970
|
-
};
|
|
971
|
-
}
|
|
972
|
-
invertedIndex[token] = {
|
|
973
|
-
docFrequency: list.df,
|
|
974
|
-
postings
|
|
975
|
-
};
|
|
976
|
-
}
|
|
977
|
-
const numeric = {};
|
|
978
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
979
|
-
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
980
|
-
}
|
|
981
|
-
const boolean = {};
|
|
982
|
-
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
983
|
-
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
984
|
-
}
|
|
985
|
-
const geopoint = {};
|
|
986
|
-
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
987
|
-
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
988
|
-
}
|
|
989
|
-
const vectorData = {};
|
|
990
|
-
for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
|
|
991
|
-
vectorData[field] = {
|
|
992
|
-
dimension: data2.dimension,
|
|
993
|
-
vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
994
|
-
hnswGraph: data2.hnsw_graph ? {
|
|
995
|
-
entryPoint: data2.hnsw_graph.entry_point,
|
|
996
|
-
maxLayer: data2.hnsw_graph.max_layer,
|
|
997
|
-
m: data2.hnsw_graph.m,
|
|
998
|
-
efConstruction: data2.hnsw_graph.ef_construction,
|
|
999
|
-
metric: validateHnswMetric(data2.hnsw_graph.metric),
|
|
1000
|
-
nodes: data2.hnsw_graph.nodes
|
|
1001
|
-
} : null
|
|
1002
|
-
};
|
|
1003
|
-
}
|
|
1004
|
-
return {
|
|
1005
|
-
indexName: raw.index_name,
|
|
1006
|
-
partitionId: raw.partition_id,
|
|
1007
|
-
totalPartitions: raw.total_partitions,
|
|
1008
|
-
language: raw.language ?? "english",
|
|
1009
|
-
schema: raw.schema ?? {},
|
|
1010
|
-
docCount: raw.doc_count ?? 0,
|
|
1011
|
-
avgDocLength: raw.avg_doc_length ?? 0,
|
|
1012
|
-
documents,
|
|
1013
|
-
invertedIndex,
|
|
1014
|
-
fieldIndexes: {
|
|
1015
|
-
numeric,
|
|
1016
|
-
boolean,
|
|
1017
|
-
enum: raw.field_indexes?.enum ?? {},
|
|
1018
|
-
geopoint
|
|
1019
|
-
},
|
|
1020
|
-
vectorData,
|
|
1021
|
-
statistics: {
|
|
1022
|
-
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
1023
|
-
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
1024
|
-
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
1025
|
-
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
1026
|
-
}
|
|
1027
|
-
};
|
|
1028
|
-
}
|
|
1029
939
|
|
|
1030
940
|
// src/runtime/detect.ts
|
|
1031
941
|
function detectRuntimeName() {
|
|
@@ -1088,10 +998,13 @@ function buildWorkerRequest(engine, hnswConfig) {
|
|
|
1088
998
|
function resolveWorkerUrl() {
|
|
1089
999
|
const base = import.meta.url;
|
|
1090
1000
|
if (base.endsWith(".mjs") || base.endsWith(".js")) {
|
|
1001
|
+
const distIndex = base.lastIndexOf("/dist/");
|
|
1002
|
+
if (distIndex !== -1) {
|
|
1003
|
+
return new URL("vector/hnsw-build-worker.mjs", base.slice(0, distIndex + 6)).href;
|
|
1004
|
+
}
|
|
1091
1005
|
return new URL("./hnsw-build-worker.mjs", base).href;
|
|
1092
1006
|
}
|
|
1093
|
-
|
|
1094
|
-
return replaced;
|
|
1007
|
+
return base.replace(/\/src\/vector\/[^/]+$/, "/dist/vector/hnsw-build-worker.mjs");
|
|
1095
1008
|
}
|
|
1096
1009
|
function spawnNodeWorker(request, onComplete, onError) {
|
|
1097
1010
|
const entryUrl = resolveWorkerUrl();
|
|
@@ -1210,6 +1123,96 @@ function createVectorPromoter(config) {
|
|
|
1210
1123
|
}
|
|
1211
1124
|
};
|
|
1212
1125
|
}
|
|
1126
|
+
function encodeRawPayloadV2(wire) {
|
|
1127
|
+
return encode(wire);
|
|
1128
|
+
}
|
|
1129
|
+
var VALID_HNSW_METRICS = /* @__PURE__ */ new Set(["cosine", "dotProduct", "euclidean"]);
|
|
1130
|
+
function validateHnswMetric(value) {
|
|
1131
|
+
if (value === void 0 || value === null) return void 0;
|
|
1132
|
+
if (typeof value === "string" && VALID_HNSW_METRICS.has(value)) {
|
|
1133
|
+
return value;
|
|
1134
|
+
}
|
|
1135
|
+
return void 0;
|
|
1136
|
+
}
|
|
1137
|
+
function deserializePayloadV2(data) {
|
|
1138
|
+
const raw = decode(data);
|
|
1139
|
+
const documents = {};
|
|
1140
|
+
for (const [docId, doc] of Object.entries(raw.documents ?? {})) {
|
|
1141
|
+
documents[docId] = {
|
|
1142
|
+
fields: doc.fields,
|
|
1143
|
+
fieldLengths: doc.field_lengths ?? {}
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1146
|
+
const fieldNames = raw.inverted_index?.field_names ?? [];
|
|
1147
|
+
const invertedIndex = {};
|
|
1148
|
+
for (const [token, list] of Object.entries(raw.inverted_index?.entries ?? {})) {
|
|
1149
|
+
const count = list.ids.length;
|
|
1150
|
+
const postings = new Array(count);
|
|
1151
|
+
for (let i = 0; i < count; i++) {
|
|
1152
|
+
postings[i] = {
|
|
1153
|
+
docId: list.ids[i],
|
|
1154
|
+
termFrequency: list.tf[i],
|
|
1155
|
+
field: fieldNames[list.fi[i]] ?? "",
|
|
1156
|
+
positions: list.pos ? list.pos[i] : []
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
|
+
invertedIndex[token] = {
|
|
1160
|
+
docFrequency: list.df,
|
|
1161
|
+
postings
|
|
1162
|
+
};
|
|
1163
|
+
}
|
|
1164
|
+
const numeric = {};
|
|
1165
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.numeric ?? {})) {
|
|
1166
|
+
numeric[field] = entries.map((e) => ({ value: e.value, docId: e.doc_id }));
|
|
1167
|
+
}
|
|
1168
|
+
const boolean = {};
|
|
1169
|
+
for (const [field, idx] of Object.entries(raw.field_indexes?.boolean ?? {})) {
|
|
1170
|
+
boolean[field] = { trueDocs: idx.true_docs ?? [], falseDocs: idx.false_docs ?? [] };
|
|
1171
|
+
}
|
|
1172
|
+
const geopoint = {};
|
|
1173
|
+
for (const [field, entries] of Object.entries(raw.field_indexes?.geopoint ?? {})) {
|
|
1174
|
+
geopoint[field] = entries.map((e) => ({ lat: e.lat, lon: e.lon, docId: e.doc_id }));
|
|
1175
|
+
}
|
|
1176
|
+
const vectorData = {};
|
|
1177
|
+
for (const [field, data2] of Object.entries(raw.vector_data ?? {})) {
|
|
1178
|
+
vectorData[field] = {
|
|
1179
|
+
dimension: data2.dimension,
|
|
1180
|
+
vectors: (data2.vectors ?? []).map((v) => ({ docId: v.doc_id, vector: v.vector })),
|
|
1181
|
+
hnswGraph: data2.hnsw_graph ? {
|
|
1182
|
+
entryPoint: data2.hnsw_graph.entry_point,
|
|
1183
|
+
maxLayer: data2.hnsw_graph.max_layer,
|
|
1184
|
+
m: data2.hnsw_graph.m,
|
|
1185
|
+
efConstruction: data2.hnsw_graph.ef_construction,
|
|
1186
|
+
metric: validateHnswMetric(data2.hnsw_graph.metric),
|
|
1187
|
+
nodes: data2.hnsw_graph.nodes
|
|
1188
|
+
} : null
|
|
1189
|
+
};
|
|
1190
|
+
}
|
|
1191
|
+
return {
|
|
1192
|
+
indexName: raw.index_name,
|
|
1193
|
+
partitionId: raw.partition_id,
|
|
1194
|
+
totalPartitions: raw.total_partitions,
|
|
1195
|
+
language: raw.language ?? "english",
|
|
1196
|
+
schema: raw.schema ?? {},
|
|
1197
|
+
docCount: raw.doc_count ?? 0,
|
|
1198
|
+
avgDocLength: raw.avg_doc_length ?? 0,
|
|
1199
|
+
documents,
|
|
1200
|
+
invertedIndex,
|
|
1201
|
+
fieldIndexes: {
|
|
1202
|
+
numeric,
|
|
1203
|
+
boolean,
|
|
1204
|
+
enum: raw.field_indexes?.enum ?? {},
|
|
1205
|
+
geopoint
|
|
1206
|
+
},
|
|
1207
|
+
vectorData,
|
|
1208
|
+
statistics: {
|
|
1209
|
+
totalDocuments: raw.statistics?.total_documents ?? 0,
|
|
1210
|
+
totalFieldLengths: raw.statistics?.total_field_lengths ?? {},
|
|
1211
|
+
averageFieldLengths: raw.statistics?.average_field_lengths ?? {},
|
|
1212
|
+
docFrequencies: raw.statistics?.doc_frequencies ?? {}
|
|
1213
|
+
}
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1213
1216
|
|
|
1214
1217
|
// src/core/document-store.ts
|
|
1215
1218
|
function createDocumentStore() {
|