@delali/narsil 0.2.4-next.54 → 0.2.4-next.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -28
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-DlYw0Ov0.d.ts → capabilities-DSsOGz6F.d.ts} +2 -2
- package/dist/{chunk-BH5SYB5W.mjs → chunk-24RXHMEL.mjs} +604 -150
- package/dist/{chunk-D2S642IV.mjs → chunk-3LHGNRYE.mjs} +1 -1
- package/dist/{chunk-TVT5V4E7.mjs → chunk-B2TFROFX.mjs} +299 -168
- package/dist/{chunk-L225SHNB.mjs → chunk-L6TNPLAR.mjs} +19 -5
- package/dist/{chunk-U2AL3SON.mjs → chunk-THYRHHO2.mjs} +3 -3
- package/dist/{chunk-NTZYA3SJ.mjs → chunk-U7KYW4BA.mjs} +3 -3
- package/dist/{chunk-FRQVMFY7.mjs → chunk-USKFU7J5.mjs} +1 -0
- package/dist/{chunk-4NVPQ6S5.mjs → chunk-UXNWQZST.mjs} +12 -3
- package/dist/{chunk-RGS7FMCB.mjs → chunk-VMRR3DQU.mjs} +74 -16
- package/dist/{chunk-2GWVQWGA.mjs → chunk-YMZCMH5M.mjs} +45 -1
- package/dist/{chunk-UTOFOJOR.mjs → chunk-YNVIUGF5.mjs} +6 -2
- package/dist/{chunk-5T6AOR37.mjs → chunk-ZNZBXMP5.mjs} +601 -60
- package/dist/{chunk-XNQAIO74.mjs → chunk-ZZT3EAUC.mjs} +177 -309
- package/dist/{client-CQUqt9Ir.d.ts → client-BQK6Tf6A.d.ts} +3 -3
- package/dist/client.d.ts +8 -8
- package/dist/client.mjs +1 -0
- package/dist/{config-CNas6wJH.d.ts → config-BhldZgaK.d.ts} +2 -2
- package/dist/{config-bXxnJrQc.d.ts → config-DmXULPa-.d.ts} +15 -7
- package/dist/direct-executor-ALAVE7FC.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +382 -162
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-Dj4I91Jv.d.ts → engine-CIOyFozK.d.ts} +15 -12
- package/dist/index.browser.mjs +1565 -528
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +11 -11
- package/dist/invalidation/filesystem.mjs +1 -1
- package/dist/{memory-xrXOZMbw.d.ts → memory-yFaVn2xZ.d.ts} +199 -107
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +11 -11
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/{schema-mLpXZBEh.d.ts → schema-d5Afu03A.d.ts} +7 -2
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +10 -10
- package/dist/{tasks-Df73Tjgq.d.ts → tasks-CH2Dod8w.d.ts} +1 -1
- package/dist/{thread-5UP6MGED.mjs → thread-DYHEQRVY.mjs} +14 -10
- package/dist/{types-BTIVO8DI.d.ts → types-DAK7mnUZ.d.ts} +1 -1
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +4 -4
- package/dist/workers/entry.mjs +4 -4
- package/package.json +9 -9
- package/dist/direct-executor-RBA5O7EN.mjs +0 -12
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { flattenSchema, SORTABLE_TEXT_FIELD_TYPE, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS,
|
|
2
|
-
import { compareCodePoints, compareComparableKeys, toComparableSortValue,
|
|
3
|
-
import { createNarsilError, ErrorCodes, NarsilError } from './chunk-
|
|
1
|
+
import { isTextFieldType, flattenSchema, SORTABLE_TEXT_FIELD_TYPE, isGeopointOrVectorType, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, MAX_CURSOR_LENGTH, RESULT_WINDOW, DEFAULT_FUZZY_PREFIX_LENGTH, DEFAULT_PAGE_SIZE, PROTOTYPE_POLLUTION_KEYS, PREFIX_MAX_EXPANSIONS, MAX_CURSOR_ANCHOR_CODE_POINTS } from './chunk-L6TNPLAR.mjs';
|
|
2
|
+
import { compareCodePoints, sortModeOf, defaultSortMode, toReducedSortValue, readSortField, compareComparableKeys, toComparableSortValue, isSortMode, SORT_MODES, SORT_VALUE_MAX_CODE_POINTS, compareSortValues } from './chunk-YMZCMH5M.mjs';
|
|
3
|
+
import { createNarsilError, ErrorCodes, NarsilError } from './chunk-USKFU7J5.mjs';
|
|
4
4
|
|
|
5
5
|
// src/core/tokenizer/constants.ts
|
|
6
6
|
var TOKEN_CACHE_SIZE_FLOOR = 5e4;
|
|
@@ -673,34 +673,6 @@ function heapPop(heap) {
|
|
|
673
673
|
return top;
|
|
674
674
|
}
|
|
675
675
|
|
|
676
|
-
// src/search/facets.ts
|
|
677
|
-
function mergeFacets(partitionFacets) {
|
|
678
|
-
const merged = /* @__PURE__ */ new Map();
|
|
679
|
-
const bounds = /* @__PURE__ */ new Map();
|
|
680
|
-
for (const partition of partitionFacets) {
|
|
681
|
-
for (const [field, facetResult] of Object.entries(partition)) {
|
|
682
|
-
let fieldMap = merged.get(field);
|
|
683
|
-
if (!fieldMap) {
|
|
684
|
-
fieldMap = /* @__PURE__ */ new Map();
|
|
685
|
-
merged.set(field, fieldMap);
|
|
686
|
-
}
|
|
687
|
-
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
688
|
-
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
689
|
-
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
}
|
|
693
|
-
const result = {};
|
|
694
|
-
for (const [field, valueMap] of merged) {
|
|
695
|
-
const values = {};
|
|
696
|
-
for (const [value, count] of valueMap) {
|
|
697
|
-
values[value] = count;
|
|
698
|
-
}
|
|
699
|
-
result[field] = { values, count: valueMap.size, errorBound: bounds.get(field) ?? 0 };
|
|
700
|
-
}
|
|
701
|
-
return result;
|
|
702
|
-
}
|
|
703
|
-
|
|
704
676
|
// src/core/bitset.ts
|
|
705
677
|
function createBitSet(capacity) {
|
|
706
678
|
return new Uint32Array(capacity + 31 >>> 5 || 1);
|
|
@@ -762,7 +734,288 @@ function bitsetFromSet(set, capacity) {
|
|
|
762
734
|
return bs;
|
|
763
735
|
}
|
|
764
736
|
|
|
737
|
+
// src/core/partition/utils.ts
|
|
738
|
+
function getOrCreateFieldNameIndex(table, fieldName) {
|
|
739
|
+
const existing = table.indexMap.get(fieldName);
|
|
740
|
+
if (existing !== void 0) return existing;
|
|
741
|
+
const idx = table.names.length;
|
|
742
|
+
table.names.push(fieldName);
|
|
743
|
+
table.indexMap.set(fieldName, idx);
|
|
744
|
+
return idx;
|
|
745
|
+
}
|
|
746
|
+
function getNestedValue(obj, path) {
|
|
747
|
+
const parts = path.split(".");
|
|
748
|
+
let current = obj;
|
|
749
|
+
for (let i = 0; i < parts.length; i++) {
|
|
750
|
+
if (current === null || current === void 0 || typeof current !== "object") return void 0;
|
|
751
|
+
if (!Object.hasOwn(current, parts[i])) return void 0;
|
|
752
|
+
current = current[parts[i]];
|
|
753
|
+
}
|
|
754
|
+
return current;
|
|
755
|
+
}
|
|
756
|
+
function stringArraysEqual(a, b) {
|
|
757
|
+
if (a === b) return true;
|
|
758
|
+
if (!a || !b) return false;
|
|
759
|
+
if (a.length !== b.length) return false;
|
|
760
|
+
for (let i = 0; i < a.length; i++) {
|
|
761
|
+
if (a[i] !== b[i]) return false;
|
|
762
|
+
}
|
|
763
|
+
return true;
|
|
764
|
+
}
|
|
765
|
+
function textFieldsChanged(oldDoc, newDoc, flatSchema) {
|
|
766
|
+
for (const [path, fieldType] of Object.entries(flatSchema)) {
|
|
767
|
+
if (!isTextFieldType(fieldType) && fieldType !== "string[]") continue;
|
|
768
|
+
const oldVal = getNestedValue(oldDoc, path);
|
|
769
|
+
const newVal = getNestedValue(newDoc, path);
|
|
770
|
+
if (fieldType !== "string[]") {
|
|
771
|
+
if (oldVal !== newVal) return true;
|
|
772
|
+
} else {
|
|
773
|
+
if (!stringArraysEqual(oldVal, newVal)) return true;
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
return false;
|
|
777
|
+
}
|
|
778
|
+
function tokenizeOptions(options) {
|
|
779
|
+
return {
|
|
780
|
+
stem: true,
|
|
781
|
+
removeStopWords: true,
|
|
782
|
+
collectSurfaces: options?.collectSurfaces === true,
|
|
783
|
+
stopWordOverride: options?.stopWordOverride,
|
|
784
|
+
customTokenizer: options?.customTokenizer
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
function getFlatSchema(state, schema) {
|
|
788
|
+
if (state.lastSchemaRef === schema && state.flatSchemaCache) return state.flatSchemaCache;
|
|
789
|
+
state.flatSchemaCache = flattenSchema(schema);
|
|
790
|
+
state.lastSchemaRef = schema;
|
|
791
|
+
return state.flatSchemaCache;
|
|
792
|
+
}
|
|
793
|
+
function getFieldValueForDoc(docStore, docId, fieldPath) {
|
|
794
|
+
const stored = docStore.get(docId);
|
|
795
|
+
if (!stored) return void 0;
|
|
796
|
+
return getNestedValue(stored.fields, fieldPath);
|
|
797
|
+
}
|
|
798
|
+
function getFieldValueByInternalId(docStore, internalId, fieldPath) {
|
|
799
|
+
const externalId = docStore.getExternalId(internalId);
|
|
800
|
+
if (externalId === void 0) return void 0;
|
|
801
|
+
return getFieldValueForDoc(docStore, externalId, fieldPath);
|
|
802
|
+
}
|
|
803
|
+
function getAllInternalDocIds(docStore) {
|
|
804
|
+
const ids = /* @__PURE__ */ new Set();
|
|
805
|
+
for (const internalId of docStore.allInternalIds()) {
|
|
806
|
+
ids.add(internalId);
|
|
807
|
+
}
|
|
808
|
+
return ids;
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
// src/core/partition/facets.ts
|
|
812
|
+
var UNDECLARED_FIELD_TYPE = "";
|
|
813
|
+
function keptFacetValues(limit) {
|
|
814
|
+
return limit !== void 0 && limit > 0 ? Math.floor(limit) : void 0;
|
|
815
|
+
}
|
|
816
|
+
function ordinalBitsetOf(state, matched) {
|
|
817
|
+
if ("ordinalBitset" in matched) return matched.ordinalBitset;
|
|
818
|
+
const bitset = createBitSet(state.docStore.internalIdCapacity());
|
|
819
|
+
for (const docId of matched) {
|
|
820
|
+
const ordinal = state.docStore.getInternalId(docId);
|
|
821
|
+
if (ordinal !== void 0) bitsetSet(bitset, ordinal);
|
|
822
|
+
}
|
|
823
|
+
return bitset;
|
|
824
|
+
}
|
|
825
|
+
function* matchedFieldValues(state, fieldPath, matched) {
|
|
826
|
+
if ("ordinalBitset" in matched) {
|
|
827
|
+
const bits = matched.ordinalBitset;
|
|
828
|
+
for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
|
|
829
|
+
let word = bits[wordIndex];
|
|
830
|
+
if (word === 0) continue;
|
|
831
|
+
const base = wordIndex << 5;
|
|
832
|
+
while (word !== 0) {
|
|
833
|
+
const trailingZeros = Math.clz32(word & -word) ^ 31;
|
|
834
|
+
yield getFieldValueByInternalId(state.docStore, base + trailingZeros, fieldPath);
|
|
835
|
+
word &= word - 1;
|
|
836
|
+
}
|
|
837
|
+
}
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
for (const docId of matched) {
|
|
841
|
+
yield getFieldValueForDoc(state.docStore, docId, fieldPath);
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
function countValuesFromDocuments(state, fieldPath, matched) {
|
|
845
|
+
const counts = /* @__PURE__ */ new Map();
|
|
846
|
+
for (const value of matchedFieldValues(state, fieldPath, matched)) {
|
|
847
|
+
if (value === void 0 || value === null) continue;
|
|
848
|
+
if (Array.isArray(value)) {
|
|
849
|
+
for (const item of value) {
|
|
850
|
+
const key = String(item);
|
|
851
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
852
|
+
}
|
|
853
|
+
} else {
|
|
854
|
+
const key = String(value);
|
|
855
|
+
counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
return counts;
|
|
859
|
+
}
|
|
860
|
+
function countRangeFromDocuments(state, fieldPath, fieldType, range, matched) {
|
|
861
|
+
let count = 0;
|
|
862
|
+
for (const value of matchedFieldValues(state, fieldPath, matched)) {
|
|
863
|
+
if ((fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) && Array.isArray(value)) {
|
|
864
|
+
for (const v of value) {
|
|
865
|
+
if (v >= range.from && v < range.to) {
|
|
866
|
+
count++;
|
|
867
|
+
break;
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
} else if (typeof value === "number" && value >= range.from && value < range.to) {
|
|
871
|
+
count++;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
return count;
|
|
875
|
+
}
|
|
876
|
+
function countNumericRanges(state, fieldPath, fieldType, ranges, bitset, matched) {
|
|
877
|
+
const counts = /* @__PURE__ */ new Map();
|
|
878
|
+
const index = state.numericIndexes.get(fieldPath);
|
|
879
|
+
for (const range of ranges) {
|
|
880
|
+
const label = `${range.from}-${range.to}`;
|
|
881
|
+
if (index === void 0) {
|
|
882
|
+
counts.set(label, countRangeFromDocuments(state, fieldPath, fieldType, range, matched));
|
|
883
|
+
} else if (Number.isNaN(range.from) || Number.isNaN(range.to)) {
|
|
884
|
+
counts.set(label, 0);
|
|
885
|
+
} else {
|
|
886
|
+
counts.set(label, index.facetRangeCount(range.from, range.to, bitset()));
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
return counts;
|
|
890
|
+
}
|
|
891
|
+
function countFieldValues(state, fieldPath, fieldType, bitset, matched) {
|
|
892
|
+
if (fieldType === "enum" || fieldType === "enum[]") {
|
|
893
|
+
const index = state.enumIndexes.get(fieldPath);
|
|
894
|
+
if (index !== void 0) return index.facetCounts(bitset());
|
|
895
|
+
} else if (fieldType === "boolean" || fieldType === "boolean[]") {
|
|
896
|
+
const index = state.booleanIndexes.get(fieldPath);
|
|
897
|
+
if (index !== void 0) {
|
|
898
|
+
const { trueCount, falseCount } = index.facetCounts(bitset());
|
|
899
|
+
const counts = /* @__PURE__ */ new Map();
|
|
900
|
+
if (trueCount > 0) counts.set("true", trueCount);
|
|
901
|
+
if (falseCount > 0) counts.set("false", falseCount);
|
|
902
|
+
return counts;
|
|
903
|
+
}
|
|
904
|
+
} else if (fieldType === "number" || fieldType === "number[]") {
|
|
905
|
+
const index = state.numericIndexes.get(fieldPath);
|
|
906
|
+
if (index !== void 0) {
|
|
907
|
+
const counts = /* @__PURE__ */ new Map();
|
|
908
|
+
for (const [value, count] of index.facetValueCounts(bitset())) {
|
|
909
|
+
counts.set(String(value), count);
|
|
910
|
+
}
|
|
911
|
+
return counts;
|
|
912
|
+
}
|
|
913
|
+
}
|
|
914
|
+
return countValuesFromDocuments(state, fieldPath, matched);
|
|
915
|
+
}
|
|
916
|
+
function computeFacets(state, matched, config, schema) {
|
|
917
|
+
const result = {};
|
|
918
|
+
const flatSchema = getFlatSchema(state, schema);
|
|
919
|
+
let matchedBitset = null;
|
|
920
|
+
const bitset = () => {
|
|
921
|
+
if (matchedBitset === null) matchedBitset = ordinalBitsetOf(state, matched);
|
|
922
|
+
return matchedBitset;
|
|
923
|
+
};
|
|
924
|
+
for (const [fieldPath, facetOpts] of Object.entries(config)) {
|
|
925
|
+
const fieldType = flatSchema[fieldPath] ?? UNDECLARED_FIELD_TYPE;
|
|
926
|
+
const valueCounts = facetOpts.ranges && (fieldType === "number" || fieldType === "number[]" || fieldType === UNDECLARED_FIELD_TYPE) ? countNumericRanges(state, fieldPath, fieldType, facetOpts.ranges, bitset, matched) : countFieldValues(state, fieldPath, fieldType, bitset, matched);
|
|
927
|
+
const ordered = Array.from(valueCounts.entries());
|
|
928
|
+
const sortDir = facetOpts.sort ?? "desc";
|
|
929
|
+
ordered.sort((a, b) => (sortDir === "asc" ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
930
|
+
const keep = Math.min(keptFacetValues(facetOpts.limit) ?? ordered.length, ordered.length);
|
|
931
|
+
const entries = ordered.slice(0, keep);
|
|
932
|
+
let errorBound = 0;
|
|
933
|
+
for (let index = keep; index < ordered.length; index++) {
|
|
934
|
+
if (ordered[index][1] > errorBound) errorBound = ordered[index][1];
|
|
935
|
+
}
|
|
936
|
+
const values = {};
|
|
937
|
+
for (const [key, count] of entries) {
|
|
938
|
+
values[key] = count;
|
|
939
|
+
}
|
|
940
|
+
result[fieldPath] = { values, count: entries.length, errorBound };
|
|
941
|
+
}
|
|
942
|
+
return result;
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
// src/search/oversample.ts
|
|
946
|
+
function oversampledShardSize(size) {
|
|
947
|
+
return Math.ceil(size * 1.5) + 10;
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
// src/search/facets.ts
|
|
951
|
+
function keptValueCount(fieldConfig) {
|
|
952
|
+
return keptFacetValues(fieldConfig?.limit);
|
|
953
|
+
}
|
|
954
|
+
function everyValueFacetConfig(config) {
|
|
955
|
+
const widened = {};
|
|
956
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
957
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
958
|
+
}
|
|
959
|
+
return widened;
|
|
960
|
+
}
|
|
961
|
+
function oversampledFacetConfig(config) {
|
|
962
|
+
const widened = {};
|
|
963
|
+
for (const [field, fieldConfig] of Object.entries(config)) {
|
|
964
|
+
if (fieldConfig?.ranges !== void 0 || fieldConfig?.sort === "asc") {
|
|
965
|
+
widened[field] = { ...fieldConfig, limit: void 0 };
|
|
966
|
+
continue;
|
|
967
|
+
}
|
|
968
|
+
const kept = keptValueCount(fieldConfig);
|
|
969
|
+
widened[field] = kept === void 0 ? fieldConfig : { ...fieldConfig, limit: oversampledShardSize(kept) };
|
|
970
|
+
}
|
|
971
|
+
return widened;
|
|
972
|
+
}
|
|
973
|
+
function mergeFacets(partitionFacets, config) {
|
|
974
|
+
const merged = /* @__PURE__ */ new Map();
|
|
975
|
+
const bounds = /* @__PURE__ */ new Map();
|
|
976
|
+
for (const partition of partitionFacets) {
|
|
977
|
+
for (const [field, facetResult] of Object.entries(partition)) {
|
|
978
|
+
let fieldMap = merged.get(field);
|
|
979
|
+
if (!fieldMap) {
|
|
980
|
+
fieldMap = /* @__PURE__ */ new Map();
|
|
981
|
+
merged.set(field, fieldMap);
|
|
982
|
+
}
|
|
983
|
+
bounds.set(field, (bounds.get(field) ?? 0) + facetResult.errorBound);
|
|
984
|
+
for (const [value, count] of Object.entries(facetResult.values)) {
|
|
985
|
+
fieldMap.set(value, (fieldMap.get(value) ?? 0) + count);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
const result = {};
|
|
990
|
+
for (const [field, valueMap] of merged) {
|
|
991
|
+
const fieldConfig = config[field];
|
|
992
|
+
const ascending = fieldConfig?.sort === "asc";
|
|
993
|
+
const ordered = Array.from(valueMap.entries());
|
|
994
|
+
ordered.sort((a, b) => (ascending ? a[1] - b[1] : b[1] - a[1]) || compareCodePoints(a[0], b[0]));
|
|
995
|
+
const kept = Math.min(keptValueCount(fieldConfig) ?? ordered.length, ordered.length);
|
|
996
|
+
let largestLeftOut = 0;
|
|
997
|
+
for (let index = kept; index < ordered.length; index++) {
|
|
998
|
+
if (ordered[index][1] > largestLeftOut) largestLeftOut = ordered[index][1];
|
|
999
|
+
}
|
|
1000
|
+
const errorBound = (bounds.get(field) ?? 0) + largestLeftOut;
|
|
1001
|
+
const values = {};
|
|
1002
|
+
for (let index = 0; index < kept; index++) {
|
|
1003
|
+
values[ordered[index][0]] = ordered[index][1];
|
|
1004
|
+
}
|
|
1005
|
+
result[field] = { values, count: kept, errorBound };
|
|
1006
|
+
}
|
|
1007
|
+
return result;
|
|
1008
|
+
}
|
|
1009
|
+
|
|
765
1010
|
// src/core/fuzzy.ts
|
|
1011
|
+
function fuzzyTermMatches(queryTerm, indexTerm, tolerance, prefixLength) {
|
|
1012
|
+
if (queryTerm === indexTerm) return true;
|
|
1013
|
+
if (tolerance <= 0) return false;
|
|
1014
|
+
if (prefixLength > 0 && queryTerm.length >= prefixLength && !indexTerm.startsWith(queryTerm.slice(0, prefixLength))) {
|
|
1015
|
+
return false;
|
|
1016
|
+
}
|
|
1017
|
+
return boundedLevenshtein(queryTerm, indexTerm, tolerance).withinTolerance;
|
|
1018
|
+
}
|
|
766
1019
|
function boundedLevenshtein(a, b, tolerance) {
|
|
767
1020
|
if (tolerance < 0) return { distance: -1, withinTolerance: false };
|
|
768
1021
|
if (a === b) return { distance: 0, withinTolerance: true };
|
|
@@ -827,6 +1080,234 @@ function decodeCursorText(encoded) {
|
|
|
827
1080
|
return new TextDecoder().decode(bytes);
|
|
828
1081
|
}
|
|
829
1082
|
|
|
1083
|
+
// src/geo/constants.ts
|
|
1084
|
+
var DEG_TO_RAD = Math.PI / 180;
|
|
1085
|
+
var MIN_POLYGON_POINTS = 3;
|
|
1086
|
+
var ANTIMERIDIAN_SPAN_DEGREES = 180;
|
|
1087
|
+
var FULL_TURN_DEGREES = 360;
|
|
1088
|
+
var VINCENTY_MAX_ITERATIONS = 200;
|
|
1089
|
+
var VINCENTY_CONVERGENCE_THRESHOLD = 1e-12;
|
|
1090
|
+
|
|
1091
|
+
// src/geo/polygon.ts
|
|
1092
|
+
function requirePolygonRing(points) {
|
|
1093
|
+
if (!Array.isArray(points) || points.length < MIN_POLYGON_POINTS) {
|
|
1094
|
+
throw new NarsilError(
|
|
1095
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
1096
|
+
`A polygon encloses an area only with at least ${MIN_POLYGON_POINTS} points`,
|
|
1097
|
+
{ points: Array.isArray(points) ? points.length : typeof points, minimum: MIN_POLYGON_POINTS }
|
|
1098
|
+
);
|
|
1099
|
+
}
|
|
1100
|
+
points.forEach((point, position) => {
|
|
1101
|
+
const { lat, lon } = point ?? {};
|
|
1102
|
+
if (typeof lat === "number" && Number.isFinite(lat) && typeof lon === "number" && Number.isFinite(lon)) return;
|
|
1103
|
+
throw new NarsilError(
|
|
1104
|
+
ErrorCodes.SEARCH_INVALID_FILTER,
|
|
1105
|
+
`A polygon point carries a finite "lat" and "lon", and the point at position ${position} does not`,
|
|
1106
|
+
{ position }
|
|
1107
|
+
);
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
function signedArea(polygon) {
|
|
1111
|
+
let area = 0;
|
|
1112
|
+
for (let i = 0, j = polygon.length - 1; i < polygon.length; j = i++) {
|
|
1113
|
+
area += polygon[j].lon * polygon[i].lat - polygon[i].lon * polygon[j].lat;
|
|
1114
|
+
}
|
|
1115
|
+
return area;
|
|
1116
|
+
}
|
|
1117
|
+
function crossesAntimeridian(polygon) {
|
|
1118
|
+
let west = Number.POSITIVE_INFINITY;
|
|
1119
|
+
let east = Number.NEGATIVE_INFINITY;
|
|
1120
|
+
for (const point of polygon) {
|
|
1121
|
+
if (point.lon < west) west = point.lon;
|
|
1122
|
+
if (point.lon > east) east = point.lon;
|
|
1123
|
+
}
|
|
1124
|
+
return east - west >= ANTIMERIDIAN_SPAN_DEGREES && signedArea(polygon) < 0;
|
|
1125
|
+
}
|
|
1126
|
+
function eastward(lon) {
|
|
1127
|
+
return lon < 0 ? lon + FULL_TURN_DEGREES : lon;
|
|
1128
|
+
}
|
|
1129
|
+
function polygonContainment(polygon) {
|
|
1130
|
+
const n = polygon.length;
|
|
1131
|
+
if (n < MIN_POLYGON_POINTS) return () => false;
|
|
1132
|
+
const unwrap = crossesAntimeridian(polygon);
|
|
1133
|
+
const lats = new Float64Array(n);
|
|
1134
|
+
const lons = new Float64Array(n);
|
|
1135
|
+
for (let i = 0; i < n; i++) {
|
|
1136
|
+
lats[i] = polygon[i].lat;
|
|
1137
|
+
lons[i] = unwrap ? eastward(polygon[i].lon) : polygon[i].lon;
|
|
1138
|
+
}
|
|
1139
|
+
return (lat, rawLon) => {
|
|
1140
|
+
const lon = unwrap ? eastward(rawLon) : rawLon;
|
|
1141
|
+
let inside = false;
|
|
1142
|
+
for (let i = 0, j = n - 1; i < n; j = i++) {
|
|
1143
|
+
const lonI = lons[i];
|
|
1144
|
+
const lonJ = lons[j];
|
|
1145
|
+
if (lonI > lon !== lonJ > lon) {
|
|
1146
|
+
const intersectLat = lats[i] + (lats[j] - lats[i]) / (lonJ - lonI) * (lon - lonI);
|
|
1147
|
+
if (lat < intersectLat) inside = !inside;
|
|
1148
|
+
}
|
|
1149
|
+
}
|
|
1150
|
+
return inside;
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
// src/filters/constants.ts
|
|
1155
|
+
var MAX_FILTER_DEPTH = 30;
|
|
1156
|
+
|
|
1157
|
+
// src/filters/operands.ts
|
|
1158
|
+
var RANGE_OPERATORS = ["gt", "lt", "gte", "lte"];
|
|
1159
|
+
var TEXT_OPERATORS = ["startsWith", "endsWith"];
|
|
1160
|
+
var EQUALITY_OPERATORS = ["eq", "ne"];
|
|
1161
|
+
var LIST_OPERATORS = ["in", "nin"];
|
|
1162
|
+
function operandKindOf(type) {
|
|
1163
|
+
if (type === void 0) return void 0;
|
|
1164
|
+
if (type === "number" || type === "number[]") return "number";
|
|
1165
|
+
if (type === "boolean" || type === "boolean[]") return "boolean";
|
|
1166
|
+
if (type === "geopoint") return "geopoint";
|
|
1167
|
+
if (type.startsWith("vector[")) return "vector";
|
|
1168
|
+
return "text";
|
|
1169
|
+
}
|
|
1170
|
+
function isFiniteNumber(value) {
|
|
1171
|
+
return typeof value === "number" && Number.isFinite(value);
|
|
1172
|
+
}
|
|
1173
|
+
function kindOfValue(value) {
|
|
1174
|
+
if (isFiniteNumber(value)) return "number";
|
|
1175
|
+
if (typeof value === "boolean") return "boolean";
|
|
1176
|
+
if (typeof value === "string") return "text";
|
|
1177
|
+
return void 0;
|
|
1178
|
+
}
|
|
1179
|
+
function invalid(message, fieldPath, operator) {
|
|
1180
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message, { fieldPath, operator });
|
|
1181
|
+
}
|
|
1182
|
+
function requireFieldKind(fieldPath, operator, type, accepted) {
|
|
1183
|
+
const kind = operandKindOf(type);
|
|
1184
|
+
if (kind === void 0 || accepted.includes(kind)) return;
|
|
1185
|
+
invalid(
|
|
1186
|
+
`The engine applies "${operator}" to ${accepted.join(" or ")} fields alone, and the schema declares field "${fieldPath}" as "${type}"`,
|
|
1187
|
+
fieldPath,
|
|
1188
|
+
operator
|
|
1189
|
+
);
|
|
1190
|
+
}
|
|
1191
|
+
function requireRadius(fieldPath, radius) {
|
|
1192
|
+
const circle = radius;
|
|
1193
|
+
if (circle === null || typeof circle !== "object") {
|
|
1194
|
+
invalid(`"radius" on field "${fieldPath}" must be an object`, fieldPath, "radius");
|
|
1195
|
+
}
|
|
1196
|
+
for (const key of ["lat", "lon", "distance"]) {
|
|
1197
|
+
if (!isFiniteNumber(circle[key])) {
|
|
1198
|
+
invalid(`"radius.${key}" on field "${fieldPath}" must be a finite number`, fieldPath, "radius");
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
if (circle.distance < 0) {
|
|
1202
|
+
invalid(
|
|
1203
|
+
`"radius.distance" on field "${fieldPath}" must be zero or more, and this filter sets ${String(circle.distance)}`,
|
|
1204
|
+
fieldPath,
|
|
1205
|
+
"radius"
|
|
1206
|
+
);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
function invalidShape(message) {
|
|
1210
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_FILTER, message);
|
|
1211
|
+
}
|
|
1212
|
+
function isObjectRecord(value) {
|
|
1213
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
1214
|
+
}
|
|
1215
|
+
function undeclaredFieldOnStrictIndex(field, option) {
|
|
1216
|
+
return new NarsilError(
|
|
1217
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
1218
|
+
`The index is strict and its schema declares no field "${field}", so no stored document has a value in that field for this ${option}`,
|
|
1219
|
+
{ field, option }
|
|
1220
|
+
);
|
|
1221
|
+
}
|
|
1222
|
+
function requireValidExpression(expression, rules, depth) {
|
|
1223
|
+
if (depth > MAX_FILTER_DEPTH) invalidShape(`A filter expression nests at most ${MAX_FILTER_DEPTH} levels deep`);
|
|
1224
|
+
if (!isObjectRecord(expression)) invalidShape("Each filter expression must be an object");
|
|
1225
|
+
const { fields, and, or, not } = expression;
|
|
1226
|
+
if (fields !== void 0) {
|
|
1227
|
+
if (!isObjectRecord(fields)) invalidShape('"fields" in a filter expression must be an object keyed by field path');
|
|
1228
|
+
for (const [fieldPath, filter] of Object.entries(fields)) {
|
|
1229
|
+
if (rules.declaredOnly && !Object.hasOwn(rules.fieldTypes, fieldPath)) {
|
|
1230
|
+
throw undeclaredFieldOnStrictIndex(fieldPath, "filter");
|
|
1231
|
+
}
|
|
1232
|
+
if (isObjectRecord(filter)) requireValidOperands(fieldPath, filter, rules.fieldTypes[fieldPath]);
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
for (const [clause, list] of [
|
|
1236
|
+
["and", and],
|
|
1237
|
+
["or", or]
|
|
1238
|
+
]) {
|
|
1239
|
+
if (list === void 0) continue;
|
|
1240
|
+
if (!Array.isArray(list)) invalidShape(`"${clause}" in a filter expression must be a list of filter expressions`);
|
|
1241
|
+
for (const nested of list) requireValidExpression(nested, rules, depth + 1);
|
|
1242
|
+
}
|
|
1243
|
+
if (not !== void 0) requireValidExpression(not, rules, depth + 1);
|
|
1244
|
+
}
|
|
1245
|
+
function requireValidFilter(expression, fieldTypes, declaredOnly = false) {
|
|
1246
|
+
if (expression === void 0 || expression === null) return;
|
|
1247
|
+
requireValidExpression(expression, { fieldTypes, declaredOnly }, 0);
|
|
1248
|
+
}
|
|
1249
|
+
function requireValidOperands(fieldPath, filter, type) {
|
|
1250
|
+
for (const operator of RANGE_OPERATORS) {
|
|
1251
|
+
if (filter[operator] === void 0) continue;
|
|
1252
|
+
if (!isFiniteNumber(filter[operator])) {
|
|
1253
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a finite number`, fieldPath, operator);
|
|
1254
|
+
}
|
|
1255
|
+
requireFieldKind(fieldPath, operator, type, ["number"]);
|
|
1256
|
+
}
|
|
1257
|
+
if (filter.between !== void 0) {
|
|
1258
|
+
const bounds = filter.between;
|
|
1259
|
+
if (!Array.isArray(bounds) || bounds.length !== 2 || !isFiniteNumber(bounds[0]) || !isFiniteNumber(bounds[1])) {
|
|
1260
|
+
invalid(
|
|
1261
|
+
`"between" on field "${fieldPath}" must be two finite numbers, the lower bound first`,
|
|
1262
|
+
fieldPath,
|
|
1263
|
+
"between"
|
|
1264
|
+
);
|
|
1265
|
+
}
|
|
1266
|
+
requireFieldKind(fieldPath, "between", type, ["number"]);
|
|
1267
|
+
}
|
|
1268
|
+
for (const operator of TEXT_OPERATORS) {
|
|
1269
|
+
if (filter[operator] === void 0) continue;
|
|
1270
|
+
if (typeof filter[operator] !== "string") {
|
|
1271
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a string`, fieldPath, operator);
|
|
1272
|
+
}
|
|
1273
|
+
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
1274
|
+
}
|
|
1275
|
+
for (const operator of EQUALITY_OPERATORS) {
|
|
1276
|
+
if (filter[operator] === void 0) continue;
|
|
1277
|
+
const kind = kindOfValue(filter[operator]);
|
|
1278
|
+
if (kind === void 0) {
|
|
1279
|
+
invalid(
|
|
1280
|
+
`"${operator}" on field "${fieldPath}" must be a string, a finite number, or a boolean`,
|
|
1281
|
+
fieldPath,
|
|
1282
|
+
operator
|
|
1283
|
+
);
|
|
1284
|
+
}
|
|
1285
|
+
requireFieldKind(fieldPath, operator, type, [kind]);
|
|
1286
|
+
}
|
|
1287
|
+
for (const operator of LIST_OPERATORS) {
|
|
1288
|
+
if (filter[operator] === void 0) continue;
|
|
1289
|
+
const values = filter[operator];
|
|
1290
|
+
if (!Array.isArray(values) || values.some((value) => typeof value !== "string")) {
|
|
1291
|
+
invalid(`"${operator}" on field "${fieldPath}" must be a list of strings`, fieldPath, operator);
|
|
1292
|
+
}
|
|
1293
|
+
requireFieldKind(fieldPath, operator, type, ["text"]);
|
|
1294
|
+
}
|
|
1295
|
+
if (filter.size !== void 0) {
|
|
1296
|
+
const size = filter.size;
|
|
1297
|
+
if (size === null || typeof size !== "object") {
|
|
1298
|
+
invalid(`"size" on field "${fieldPath}" must be a comparison object`, fieldPath, "size");
|
|
1299
|
+
}
|
|
1300
|
+
requireValidOperands(`${fieldPath}.size`, size, "number");
|
|
1301
|
+
}
|
|
1302
|
+
if (filter.radius !== void 0) requireRadius(fieldPath, filter.radius);
|
|
1303
|
+
if (filter.polygon !== void 0) {
|
|
1304
|
+
if (!isObjectRecord(filter.polygon)) {
|
|
1305
|
+
invalid(`"polygon" on field "${fieldPath}" must be an object`, fieldPath, "polygon");
|
|
1306
|
+
}
|
|
1307
|
+
requirePolygonRing(filter.polygon.points);
|
|
1308
|
+
}
|
|
1309
|
+
}
|
|
1310
|
+
|
|
830
1311
|
// src/search/sorting.ts
|
|
831
1312
|
function isFieldList(sort) {
|
|
832
1313
|
return Array.isArray(sort);
|
|
@@ -834,29 +1315,71 @@ function isFieldList(sort) {
|
|
|
834
1315
|
function normalizeSort(sort) {
|
|
835
1316
|
if (sort === void 0) return [];
|
|
836
1317
|
if (isFieldList(sort)) {
|
|
837
|
-
return sort.map(
|
|
1318
|
+
return sort.map(
|
|
1319
|
+
(entry) => entry.mode === void 0 ? { field: entry.field, direction: entry.direction } : { field: entry.field, direction: entry.direction, mode: entry.mode }
|
|
1320
|
+
);
|
|
838
1321
|
}
|
|
839
1322
|
return Object.entries(sort).map(([field, direction]) => ({ field, direction }));
|
|
840
1323
|
}
|
|
841
|
-
function
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
1324
|
+
function sortModesOf(fields) {
|
|
1325
|
+
return fields.map(sortModeOf);
|
|
1326
|
+
}
|
|
1327
|
+
function sortSignatureEntry(entry) {
|
|
1328
|
+
const mode = sortModeOf(entry);
|
|
1329
|
+
return mode === defaultSortMode(entry.direction) ? [entry.field, entry.direction] : [entry.field, entry.direction, mode];
|
|
1330
|
+
}
|
|
1331
|
+
function requireSortMode(entry, fieldType) {
|
|
1332
|
+
if (entry.mode === void 0) return;
|
|
1333
|
+
if (!isSortMode(entry.mode)) {
|
|
1334
|
+
throw new NarsilError(
|
|
1335
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
1336
|
+
`A sort mode is one of ${SORT_MODES.map((mode) => `"${mode}"`).join(", ")}, and the mode on "${entry.field}" is "${String(entry.mode)}"`,
|
|
1337
|
+
{ field: entry.field, mode: String(entry.mode) }
|
|
1338
|
+
);
|
|
1339
|
+
}
|
|
1340
|
+
const averages = entry.mode === "avg" || entry.mode === "median";
|
|
1341
|
+
if (averages && fieldType !== void 0 && fieldType !== "number" && fieldType !== "number[]") {
|
|
847
1342
|
throw new NarsilError(
|
|
848
1343
|
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
849
|
-
`
|
|
850
|
-
{ field: entry.field, fieldType:
|
|
1344
|
+
`The engine applies the "${entry.mode}" mode to a number field alone, and "${entry.field}" is a ${fieldType} field`,
|
|
1345
|
+
{ field: entry.field, fieldType, mode: entry.mode }
|
|
851
1346
|
);
|
|
852
1347
|
}
|
|
853
1348
|
}
|
|
854
|
-
function
|
|
855
|
-
|
|
1349
|
+
function requireSortableFields(sort, schema, strict = false) {
|
|
1350
|
+
const fields = normalizeSort(sort);
|
|
1351
|
+
if (fields.length === 0) return;
|
|
1352
|
+
const flatSchema = flattenSchema(schema);
|
|
1353
|
+
for (const entry of fields) {
|
|
1354
|
+
const fieldType = flatSchema[entry.field];
|
|
1355
|
+
if (strict && fieldType === void 0) throw undeclaredFieldOnStrictIndex(entry.field, "sort");
|
|
1356
|
+
if (entry.direction !== "asc" && entry.direction !== "desc") {
|
|
1357
|
+
throw new NarsilError(
|
|
1358
|
+
ErrorCodes.SEARCH_INVALID_MODE,
|
|
1359
|
+
`A sort direction is "asc" or "desc", and the direction on "${entry.field}" is "${String(entry.direction)}"`,
|
|
1360
|
+
{ field: entry.field, direction: String(entry.direction) }
|
|
1361
|
+
);
|
|
1362
|
+
}
|
|
1363
|
+
requireSortMode(entry, fieldType);
|
|
1364
|
+
if (fieldType === "string") {
|
|
1365
|
+
throw new NarsilError(
|
|
1366
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
1367
|
+
`The engine sorts by text field "${entry.field}" only where the schema declares it "${SORTABLE_TEXT_FIELD_TYPE}", because ordering text takes far more memory per document than ordering a number`,
|
|
1368
|
+
{ field: entry.field, fieldType }
|
|
1369
|
+
);
|
|
1370
|
+
}
|
|
1371
|
+
if (isGeopointOrVectorType(fieldType)) {
|
|
1372
|
+
throw new NarsilError(
|
|
1373
|
+
ErrorCodes.SEARCH_INVALID_FIELD,
|
|
1374
|
+
`The engine sorts by a number, boolean, enum, or sortable text field, and "${entry.field}" is a ${fieldType} field`,
|
|
1375
|
+
{ field: entry.field, fieldType }
|
|
1376
|
+
);
|
|
1377
|
+
}
|
|
1378
|
+
}
|
|
856
1379
|
}
|
|
857
|
-
function readSortValues(document, fields) {
|
|
858
|
-
if (!document) return fields.map(() =>
|
|
859
|
-
return fields.map((field) =>
|
|
1380
|
+
function readSortValues(document, fields, modes) {
|
|
1381
|
+
if (!document) return fields.map(() => null);
|
|
1382
|
+
return fields.map((field, index) => toReducedSortValue(readSortField(document, field), modes[index] ?? "min"));
|
|
860
1383
|
}
|
|
861
1384
|
function applySorting(hits, sort, sortKeyOf) {
|
|
862
1385
|
const normalized = normalizeSort(sort);
|
|
@@ -874,7 +1397,7 @@ function applySorting(hits, sort, sortKeyOf) {
|
|
|
874
1397
|
}
|
|
875
1398
|
|
|
876
1399
|
// src/search/cursor.ts
|
|
877
|
-
var CURSOR_VERSION =
|
|
1400
|
+
var CURSOR_VERSION = 4;
|
|
878
1401
|
function invalidCursor(cursor, reason) {
|
|
879
1402
|
const shown = cursor.length > 100 ? `${cursor.slice(0, 100)}...` : cursor;
|
|
880
1403
|
return new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, `Failed to decode cursor: ${reason}`, { cursor: shown });
|
|
@@ -914,7 +1437,7 @@ function sortSignatureOf(sort) {
|
|
|
914
1437
|
);
|
|
915
1438
|
}
|
|
916
1439
|
}
|
|
917
|
-
return JSON.stringify(fields.map(
|
|
1440
|
+
return JSON.stringify(fields.map(sortSignatureEntry));
|
|
918
1441
|
}
|
|
919
1442
|
function encodePageCursor(cursor) {
|
|
920
1443
|
const payload = { v: CURSOR_VERSION, a: cursor.anchor };
|
|
@@ -924,6 +1447,7 @@ function encodePageCursor(cursor) {
|
|
|
924
1447
|
} else if (cursor.score !== null) {
|
|
925
1448
|
payload.s = cursor.score;
|
|
926
1449
|
}
|
|
1450
|
+
if (cursor.depth !== void 0 && cursor.depth > 0) payload.d = cursor.depth;
|
|
927
1451
|
payload.q = cursor.binding;
|
|
928
1452
|
return encodeCursorText(JSON.stringify(payload));
|
|
929
1453
|
}
|
|
@@ -969,7 +1493,7 @@ function decodePageCursor(cursor) {
|
|
|
969
1493
|
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
970
1494
|
throw invalidCursor(cursor, "expected an object");
|
|
971
1495
|
}
|
|
972
|
-
const { v, a, s, k, o, q } = parsed;
|
|
1496
|
+
const { v, a, s, k, o, d, q } = parsed;
|
|
973
1497
|
if (v !== CURSOR_VERSION) throw invalidCursor(cursor, `unsupported cursor version ${String(v)}`);
|
|
974
1498
|
if (typeof q !== "string" || !/^[0-9a-f]{8}$/.test(q)) {
|
|
975
1499
|
throw invalidCursor(cursor, '"q" must be 8 lowercase hex digits');
|
|
@@ -982,6 +1506,9 @@ function decodePageCursor(cursor) {
|
|
|
982
1506
|
throw invalidCursor(cursor, '"s" must be a finite number');
|
|
983
1507
|
}
|
|
984
1508
|
if (s !== void 0 && k !== void 0) throw invalidCursor(cursor, 'a cursor carries "s" or "k", never both');
|
|
1509
|
+
if (d !== void 0 && (typeof d !== "number" || !Number.isSafeInteger(d) || d < 1)) {
|
|
1510
|
+
throw invalidCursor(cursor, '"d" must be a positive integer');
|
|
1511
|
+
}
|
|
985
1512
|
const sortKey = k === void 0 ? null : decodeSortKey(k, cursor);
|
|
986
1513
|
if (o !== void 0 && typeof o !== "string") throw invalidCursor(cursor, '"o" must be a string');
|
|
987
1514
|
if (sortKey === null !== (o === void 0)) {
|
|
@@ -992,6 +1519,7 @@ function decodePageCursor(cursor) {
|
|
|
992
1519
|
score: typeof s === "number" ? s : null,
|
|
993
1520
|
sortKey,
|
|
994
1521
|
sortSignature: typeof o === "string" ? o : null,
|
|
1522
|
+
depth: typeof d === "number" ? d : 0,
|
|
995
1523
|
binding: q
|
|
996
1524
|
};
|
|
997
1525
|
}
|
|
@@ -1069,18 +1597,21 @@ function applyPagination(results, limit, offset, binding, cursor, sort, pinnedId
|
|
|
1069
1597
|
let nextCursor;
|
|
1070
1598
|
const hasMore = afterOffset + limit < results.length;
|
|
1071
1599
|
const lastResult = lastOrganicResult(sliced, pinnedIds);
|
|
1600
|
+
const depth = afterOffset + sliced.length;
|
|
1072
1601
|
if (hasMore && lastResult !== void 0) {
|
|
1073
1602
|
nextCursor = sort !== void 0 ? encodePageCursor({
|
|
1074
1603
|
anchor: lastResult.id,
|
|
1075
1604
|
score: null,
|
|
1076
1605
|
sortKey: sort.sortKeyOf(lastResult.id).map(toComparableSortValue),
|
|
1077
1606
|
sortSignature: sort.signature,
|
|
1607
|
+
depth,
|
|
1078
1608
|
binding
|
|
1079
1609
|
}) : encodePageCursor({
|
|
1080
1610
|
anchor: lastResult.id,
|
|
1081
1611
|
score: lastResult.score ?? null,
|
|
1082
1612
|
sortKey: null,
|
|
1083
1613
|
sortSignature: null,
|
|
1614
|
+
depth,
|
|
1084
1615
|
binding
|
|
1085
1616
|
});
|
|
1086
1617
|
}
|
|
@@ -1152,7 +1683,7 @@ function fulltextMatches(partition, params, language, schema, options) {
|
|
|
1152
1683
|
prefixExpansion: prepared.prefixExpansion,
|
|
1153
1684
|
fields: params.fields,
|
|
1154
1685
|
tolerance: params.tolerance ?? 0,
|
|
1155
|
-
prefixLength: params.prefixLength ??
|
|
1686
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
1156
1687
|
exact: params.exact ?? false,
|
|
1157
1688
|
filterBitset: prepared.filterBitset
|
|
1158
1689
|
});
|
|
@@ -1173,7 +1704,7 @@ function fulltextSearch(partition, params, language, schema, options) {
|
|
|
1173
1704
|
fields: params.fields,
|
|
1174
1705
|
boost: params.boost,
|
|
1175
1706
|
tolerance: params.tolerance ?? 0,
|
|
1176
|
-
prefixLength: params.prefixLength ??
|
|
1707
|
+
prefixLength: params.prefixLength ?? DEFAULT_FUZZY_PREFIX_LENGTH,
|
|
1177
1708
|
exact: params.exact ?? false,
|
|
1178
1709
|
bm25Params: options?.bm25Params,
|
|
1179
1710
|
globalStats: options?.globalStats,
|
|
@@ -1333,20 +1864,21 @@ async function fanOutQuery(manager, params, language, schema, config, searchOpti
|
|
|
1333
1864
|
if (config.dispatcher) {
|
|
1334
1865
|
outcomes = await dispatchWithDispatcher(partitions, params, language, schema, options, config.dispatcher);
|
|
1335
1866
|
if (params.facets) {
|
|
1336
|
-
facets = collectAndMergeFacets(outcomes, params, schema);
|
|
1867
|
+
facets = collectAndMergeFacets(outcomes, params.facets, schema);
|
|
1337
1868
|
}
|
|
1338
1869
|
} else {
|
|
1339
1870
|
outcomes = [];
|
|
1340
1871
|
const partitionFacets = [];
|
|
1872
|
+
const everyValue = params.facets ? everyValueFacetConfig(params.facets) : void 0;
|
|
1341
1873
|
for (const partition of partitions) {
|
|
1342
1874
|
const result = dispatchSinglePartition(partition, params, language, schema, options);
|
|
1343
|
-
if (
|
|
1344
|
-
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result),
|
|
1875
|
+
if (everyValue) {
|
|
1876
|
+
partitionFacets.push(partition.computeFacets(facetMatchSetOf(result), everyValue, schema));
|
|
1345
1877
|
}
|
|
1346
1878
|
outcomes.push({ result, partition });
|
|
1347
1879
|
}
|
|
1348
1880
|
if (params.facets) {
|
|
1349
|
-
facets = mergeFacets(partitionFacets);
|
|
1881
|
+
facets = mergeFacets(partitionFacets, params.facets);
|
|
1350
1882
|
}
|
|
1351
1883
|
}
|
|
1352
1884
|
const allScoredArrays = outcomes.map((o) => o.result.scored);
|
|
@@ -1384,14 +1916,13 @@ function dispatchWithDispatcher(partitions, params, language, schema, options, d
|
|
|
1384
1916
|
function dispatchSinglePartition(partition, params, language, schema, options) {
|
|
1385
1917
|
return fulltextSearch(partition, params, language, schema, options);
|
|
1386
1918
|
}
|
|
1387
|
-
function collectAndMergeFacets(outcomes,
|
|
1919
|
+
function collectAndMergeFacets(outcomes, facetConfig, schema) {
|
|
1920
|
+
const everyValue = everyValueFacetConfig(facetConfig);
|
|
1388
1921
|
const partitionFacets = [];
|
|
1389
1922
|
for (const outcome of outcomes) {
|
|
1390
|
-
|
|
1391
|
-
const facetResult = outcome.partition.computeFacets(facetMatchSetOf(outcome.result), params.facets, schema);
|
|
1392
|
-
partitionFacets.push(facetResult);
|
|
1923
|
+
partitionFacets.push(outcome.partition.computeFacets(facetMatchSetOf(outcome.result), everyValue, schema));
|
|
1393
1924
|
}
|
|
1394
|
-
return mergeFacets(partitionFacets);
|
|
1925
|
+
return mergeFacets(partitionFacets, facetConfig);
|
|
1395
1926
|
}
|
|
1396
1927
|
|
|
1397
1928
|
// src/partitioning/match-count.ts
|
|
@@ -1550,6 +2081,16 @@ function fnv1aBytes(bytes) {
|
|
|
1550
2081
|
}
|
|
1551
2082
|
return hash >>> 0;
|
|
1552
2083
|
}
|
|
2084
|
+
function fnv1aUint32s(values) {
|
|
2085
|
+
let hash = FNV_OFFSET_BASIS;
|
|
2086
|
+
for (const value of values) {
|
|
2087
|
+
for (let shift = 24; shift >= 0; shift -= 8) {
|
|
2088
|
+
hash ^= value >>> shift & 255;
|
|
2089
|
+
hash = Math.imul(hash, FNV_PRIME);
|
|
2090
|
+
}
|
|
2091
|
+
}
|
|
2092
|
+
return hash >>> 0;
|
|
2093
|
+
}
|
|
1553
2094
|
function fnv1a(input) {
|
|
1554
2095
|
return fnv1aBytes(encoder.encode(input));
|
|
1555
2096
|
}
|
|
@@ -1573,4 +2114,4 @@ function setNestedValue(obj, path, value) {
|
|
|
1573
2114
|
current[segments[segments.length - 1]] = value;
|
|
1574
2115
|
}
|
|
1575
2116
|
|
|
1576
|
-
export { applyPagination, applyProjection, applySorting, bitsetAnd, bitsetFromSet, bitsetHas, bitsetNot, bitsetOr, bitsetSet, boundedLevenshtein, clampRowCount, clearNormalizationCache, cloneProjected, collectQueryTermStats, configureNormalizationCache, countsWithoutScores, createBitSet, decodePageCursor, encodePageCursor, fanOutMatchCount, fanOutQuery, fnv1a, fnv1aBytes, fulltextMatches, getNormalizationCacheSize, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, partitionsIn, projectionKeepsField, pruneStatsToQueryTerms, readSortValues, requireMatchingCursor, requireSortableFields, requireWithinResultWindow, resetNormalizationCache, resolveProjection, sanitizeGlobalStats, setNestedValue, sortSignatureOf, tokenize, tokenizeIterator, visitTokens };
|
|
2117
|
+
export { DEG_TO_RAD, MAX_FILTER_DEPTH, MIN_POLYGON_POINTS, VINCENTY_CONVERGENCE_THRESHOLD, VINCENTY_MAX_ITERATIONS, applyPagination, applyProjection, applySorting, bitsetAnd, bitsetFromSet, bitsetHas, bitsetNot, bitsetOr, bitsetSet, boundedLevenshtein, clampRowCount, clearNormalizationCache, cloneProjected, collectQueryTermStats, computeFacets, configureNormalizationCache, countsWithoutScores, createBitSet, decodePageCursor, encodePageCursor, everyValueFacetConfig, fanOutMatchCount, fanOutQuery, fnv1a, fnv1aBytes, fnv1aUint32s, fulltextMatches, fuzzyTermMatches, getAllInternalDocIds, getFieldValueByInternalId, getFlatSchema, getNestedValue, getNormalizationCacheSize, getOrCreateFieldNameIndex, kWayMerge, mergeFacets, mergePartitionStats, normalizeSort, oversampledFacetConfig, oversampledShardSize, partitionsIn, polygonContainment, projectionKeepsField, pruneStatsToQueryTerms, readSortValues, requireMatchingCursor, requirePolygonRing, requireSortableFields, requireValidFilter, requireValidOperands, requireWithinResultWindow, resetNormalizationCache, resolveProjection, sanitizeGlobalStats, setNestedValue, sortModesOf, sortSignatureEntry, sortSignatureOf, textFieldsChanged, tokenize, tokenizeIterator, tokenizeOptions, undeclaredFieldOnStrictIndex, visitTokens };
|