@delali/narsil 0.3.1-next.163 → 0.3.1-next.169
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
- package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
- package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
- package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
- package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
- package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
- package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
- package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
- package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
- package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
- package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
- package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
- package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
- package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
- package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
- package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
- package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
- package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
- package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
- package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
- package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
- package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
- package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
- package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
- package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
- package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
- package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
- package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
- package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
- package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
- package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
- package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
- package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
- package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
- package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
- package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
- package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
- package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
- package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
- package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
- package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
- package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
- package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
- package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
- package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
- package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
- package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
- package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
- package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
- package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
- package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
- package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
- package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
- package/dist/client.d.ts +8 -8
- package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
- package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
- package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
- package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
- package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution/transport/tcp.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +39 -39
- package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
- package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
- package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +7 -7
- package/dist/index.react-native.mjs +6 -6
- package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
- package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
- package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
- package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
- package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
- package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
- package/dist/partial-words.browser.mjs +3 -3
- package/dist/partial-words.mjs +7 -7
- package/dist/pattern.browser.mjs +2 -2
- package/dist/pattern.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
- package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
- package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
- package/dist/query-suggestions/merge-worker.mjs +10 -10
- package/dist/query-suggestions.browser.mjs +3 -3
- package/dist/query-suggestions.d.ts +2 -2
- package/dist/query-suggestions.mjs +8 -8
- package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
- package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
- package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
- package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
- package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
- package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
- package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
- package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
- package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
- package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
- package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
- package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
- package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
- package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
- package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
- package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
- package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
- package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
- package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
- package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
- package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
- package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
- package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
- package/dist/react-native/partial-words.mjs +6 -6
- package/dist/react-native/pattern.mjs +3 -3
- package/dist/react-native/query-suggestions.mjs +6 -6
- package/dist/react-native/sync.mjs +6 -6
- package/dist/react-native/synonyms.mjs +6 -6
- package/dist/react.d.ts +9 -9
- package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
- package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
- package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
- package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
- package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
- package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +26 -26
- package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
- package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
- package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
- package/dist/sync/filesystem.d.ts +3 -3
- package/dist/sync/filesystem.mjs +15 -15
- package/dist/sync/indexeddb.browser.mjs +1 -1
- package/dist/sync/indexeddb.d.ts +3 -3
- package/dist/sync/indexeddb.mjs +2 -2
- package/dist/sync.browser.mjs +3 -3
- package/dist/sync.d.ts +4 -4
- package/dist/sync.mjs +7 -7
- package/dist/synonyms.browser.mjs +3 -3
- package/dist/synonyms.mjs +7 -7
- package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
- package/dist/thread-OWXVP2ZA.mjs +30 -0
- package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
- package/dist/workers/entry.mjs +13 -13
- package/package.json +9 -8
- package/dist/thread-XWDK6TAR.mjs +0 -30
- /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
|
@@ -1646,6 +1646,7 @@ var extractVectorFromDoc;
|
|
|
1646
1646
|
var fnv1a;
|
|
1647
1647
|
var generateId;
|
|
1648
1648
|
var getLanguage;
|
|
1649
|
+
var getNestedValue;
|
|
1649
1650
|
var getVectorFieldPaths;
|
|
1650
1651
|
var indexChangeOutsideExternalWaits;
|
|
1651
1652
|
var insertDocument;
|
|
@@ -1720,6 +1721,7 @@ function bindOptionalWritesHost(host, segmentCodec) {
|
|
|
1720
1721
|
fnv1a,
|
|
1721
1722
|
generateId,
|
|
1722
1723
|
getLanguage,
|
|
1724
|
+
getNestedValue,
|
|
1723
1725
|
getVectorFieldPaths,
|
|
1724
1726
|
indexChangeOutsideExternalWaits,
|
|
1725
1727
|
insertDocument,
|
|
@@ -1886,21 +1888,26 @@ function recoveryFailure(metadata, err) {
|
|
|
1886
1888
|
stopWords: metadata.stopWords
|
|
1887
1889
|
});
|
|
1888
1890
|
}
|
|
1889
|
-
async function
|
|
1890
|
-
|
|
1891
|
-
|
|
1891
|
+
async function savedPartitionCountOf(metadata) {
|
|
1892
|
+
const count = metadata.partitionCount;
|
|
1893
|
+
if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 1) {
|
|
1894
|
+
const held = typeof count === "number" ? String(count) : `a ${typeof count} as its count of`;
|
|
1895
|
+
throw new NarsilError(
|
|
1896
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
1897
|
+
`Recovery of index "${metadata.indexName}" failed: its metadata holds ${held} partitions, and an index holds a whole number above zero`,
|
|
1898
|
+
{ indexName: metadata.indexName, partitionCount: typeof count === "number" ? count : typeof count }
|
|
1899
|
+
);
|
|
1892
1900
|
}
|
|
1893
|
-
const
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
while (manager.partitionCount < metadata.partitionCount) manager.addPartition();
|
|
1897
|
-
if (manager.partitionCount > metadata.partitionCount) {
|
|
1898
|
-
manager.setPartitions(manager.getAllPartitions().slice(0, metadata.partitionCount));
|
|
1899
|
-
}
|
|
1900
|
-
});
|
|
1901
|
+
const { requireSavedPartitionCount } = await loadStorageCode();
|
|
1902
|
+
requireSavedPartitionCount(count, "metadata", metadata.indexName);
|
|
1903
|
+
return count;
|
|
1901
1904
|
}
|
|
1902
1905
|
async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
1903
|
-
const loadingSchemaCode =
|
|
1906
|
+
const loadingSchemaCode = loadSchemaCodeFor(
|
|
1907
|
+
loadData ? metadata.schema : null,
|
|
1908
|
+
metadata.tokenizer,
|
|
1909
|
+
metadata.pendingAnalysis?.tokenizer
|
|
1910
|
+
);
|
|
1904
1911
|
if (loadingSchemaCode !== null) await loadingSchemaCode;
|
|
1905
1912
|
const existing = deps.indexRegistry.get(metadata.indexName);
|
|
1906
1913
|
if (existing !== void 0) {
|
|
@@ -1909,14 +1916,15 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1909
1916
|
adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
|
|
1910
1917
|
}
|
|
1911
1918
|
if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
|
|
1919
|
+
const partitionCount2 = await savedPartitionCountOf(metadata);
|
|
1912
1920
|
deps.executor.createIndex(
|
|
1913
1921
|
metadata.indexName,
|
|
1914
1922
|
existing.config,
|
|
1915
1923
|
existing.language,
|
|
1916
1924
|
false,
|
|
1917
|
-
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
|
|
1925
|
+
lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language),
|
|
1926
|
+
partitionCount2
|
|
1918
1927
|
);
|
|
1919
|
-
await restorePartitionCount(deps, metadata);
|
|
1920
1928
|
}
|
|
1921
1929
|
return;
|
|
1922
1930
|
}
|
|
@@ -1929,6 +1937,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1929
1937
|
} catch (err) {
|
|
1930
1938
|
throw recoveryFailure(metadata, err);
|
|
1931
1939
|
}
|
|
1940
|
+
const partitionCount = await savedPartitionCountOf(metadata);
|
|
1932
1941
|
const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
|
|
1933
1942
|
const prepared = prepareStoredSynonyms(
|
|
1934
1943
|
metadata.synonyms,
|
|
@@ -1941,8 +1950,14 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1941
1950
|
validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
|
|
1942
1951
|
validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
|
|
1943
1952
|
if (loadData) {
|
|
1944
|
-
deps.executor.createIndex(
|
|
1945
|
-
|
|
1953
|
+
deps.executor.createIndex(
|
|
1954
|
+
metadata.indexName,
|
|
1955
|
+
indexConfig,
|
|
1956
|
+
language,
|
|
1957
|
+
false,
|
|
1958
|
+
prepared?.lookup ?? null,
|
|
1959
|
+
partitionCount
|
|
1960
|
+
);
|
|
1946
1961
|
}
|
|
1947
1962
|
} catch (err) {
|
|
1948
1963
|
throw recoveryFailure(metadata, err);
|
|
@@ -1956,7 +1971,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1956
1971
|
indexUuid: metadata.indexUuid ?? null,
|
|
1957
1972
|
heldPartitions: metadata.heldPartitions ?? null,
|
|
1958
1973
|
documentCount: metadata.documentCount ?? 0,
|
|
1959
|
-
partitionCount
|
|
1974
|
+
partitionCount,
|
|
1960
1975
|
analysisRevision: metadata.analysisRevision ?? null,
|
|
1961
1976
|
arrayPositionGap: metadata.arrayPositionGap ?? null,
|
|
1962
1977
|
synonyms,
|
|
@@ -1979,6 +1994,20 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
|
|
|
1979
1994
|
}
|
|
1980
1995
|
|
|
1981
1996
|
// src/engine/mutations/durable-rollback.ts
|
|
1997
|
+
function holdVectorsForRollback(indexes, docId) {
|
|
1998
|
+
const releases = [];
|
|
1999
|
+
for (const index of indexes.values()) releases.push(index.holdForRollback(docId));
|
|
2000
|
+
return () => {
|
|
2001
|
+
for (const release of releases) release();
|
|
2002
|
+
};
|
|
2003
|
+
}
|
|
2004
|
+
function restoreDocumentVectors(docId, vectors, indexes, partitionId) {
|
|
2005
|
+
for (const [fieldPath, index] of indexes) {
|
|
2006
|
+
const vector = vectors.get(fieldPath);
|
|
2007
|
+
if (vector !== void 0) index.restore(docId, vector, partitionId);
|
|
2008
|
+
else if (index.has(docId)) index.remove(docId);
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
1982
2011
|
function rollbackFailed(docId, originalError, rollbackError) {
|
|
1983
2012
|
throw new NarsilError(
|
|
1984
2013
|
ErrorCodes.REPLICATION_ROLLBACK_FAILED,
|
|
@@ -2027,7 +2056,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
2027
2056
|
manager.insert(docId, previous.partitionDoc, {
|
|
2028
2057
|
preparedText: { after: preparedText?.before }
|
|
2029
2058
|
});
|
|
2030
|
-
|
|
2059
|
+
restoreDocumentVectors(
|
|
2031
2060
|
docId,
|
|
2032
2061
|
previous.extractedVectors,
|
|
2033
2062
|
manager.getVectorIndexes(),
|
|
@@ -2061,8 +2090,12 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
|
|
|
2061
2090
|
skipClone: rollbackRef !== void 0,
|
|
2062
2091
|
preparedText: { before: preparedText?.after, after: preparedText?.before }
|
|
2063
2092
|
});
|
|
2064
|
-
|
|
2065
|
-
|
|
2093
|
+
restoreDocumentVectors(
|
|
2094
|
+
docId,
|
|
2095
|
+
previous.extractedVectors,
|
|
2096
|
+
manager.getVectorIndexes(),
|
|
2097
|
+
manager.partitionIdOf(docId)
|
|
2098
|
+
);
|
|
2066
2099
|
ctx.orchestrator.publishToWorkers({
|
|
2067
2100
|
type: "update",
|
|
2068
2101
|
indexName,
|
|
@@ -2761,6 +2794,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
|
|
|
2761
2794
|
return queued;
|
|
2762
2795
|
}
|
|
2763
2796
|
prepared.restoreDoc = ctx.durability ? manager.get(docId) : void 0;
|
|
2797
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
2764
2798
|
manager.remove(docId, void 0, {
|
|
2765
2799
|
onRemovedSpellings: (spellings) => {
|
|
2766
2800
|
prepared.removedSpellings = spellings;
|
|
@@ -2801,6 +2835,8 @@ async function removeDocument2(ctx, indexName, docId, options) {
|
|
|
2801
2835
|
} catch (err) {
|
|
2802
2836
|
await rollbackRemovedDocument(ctx, indexName, docId, prepared.restoreDoc, err, prepared.preparedText);
|
|
2803
2837
|
throw err;
|
|
2838
|
+
} finally {
|
|
2839
|
+
prepared.releaseVectorHolds?.();
|
|
2804
2840
|
}
|
|
2805
2841
|
} else {
|
|
2806
2842
|
await apply();
|
|
@@ -2829,29 +2865,35 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
2829
2865
|
}
|
|
2830
2866
|
}
|
|
2831
2867
|
const applies = prepared.map((removal) => applyOfRemoval(ctx, indexName, removal));
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
|
|
2836
|
-
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2842
|
-
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2868
|
+
try {
|
|
2869
|
+
const failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
2870
|
+
for (let i = 0; i < prepared.length; i++) {
|
|
2871
|
+
const removal = prepared[i];
|
|
2872
|
+
const failure = failures[i];
|
|
2873
|
+
if (failure !== null) {
|
|
2874
|
+
let error = failure.error;
|
|
2875
|
+
try {
|
|
2876
|
+
await rollbackRemovedDocument(
|
|
2877
|
+
ctx,
|
|
2878
|
+
indexName,
|
|
2879
|
+
removal.docId,
|
|
2880
|
+
removal.restoreDoc,
|
|
2881
|
+
error,
|
|
2882
|
+
removal.preparedText
|
|
2883
|
+
);
|
|
2884
|
+
} catch (rollbackError) {
|
|
2885
|
+
error = rollbackError;
|
|
2886
|
+
}
|
|
2887
|
+
removal.releaseVectorHolds?.();
|
|
2888
|
+
failed.push({ docId: removal.docId, error: asRemoveError(error) });
|
|
2889
|
+
continue;
|
|
2849
2890
|
}
|
|
2850
|
-
|
|
2851
|
-
|
|
2891
|
+
removal.releaseVectorHolds?.();
|
|
2892
|
+
if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
|
|
2893
|
+
succeeded.push(removal.docId);
|
|
2852
2894
|
}
|
|
2853
|
-
|
|
2854
|
-
|
|
2895
|
+
} finally {
|
|
2896
|
+
for (const removal of prepared) removal.releaseVectorHolds?.();
|
|
2855
2897
|
}
|
|
2856
2898
|
if (chunkEnd < docIds.length) {
|
|
2857
2899
|
await new Promise((r) => setTimeout(r, 0));
|
|
@@ -2866,9 +2908,6 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
|
|
|
2866
2908
|
}
|
|
2867
2909
|
|
|
2868
2910
|
// src/engine/mutations/update.ts
|
|
2869
|
-
function extractVectorFromDocForUpdate(document, fieldPath) {
|
|
2870
|
-
return extractVectorFromDoc(document, fieldPath);
|
|
2871
|
-
}
|
|
2872
2911
|
function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
2873
2912
|
if (extractedVectors.size === 0) {
|
|
2874
2913
|
return { partitionDoc: document };
|
|
@@ -2892,8 +2931,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
|
|
|
2892
2931
|
for (const fieldPath of extractedVectors.keys()) delete source[fieldPath];
|
|
2893
2932
|
}
|
|
2894
2933
|
const partitionDoc = structuredClone(source);
|
|
2895
|
-
for (const fieldPath of extractedVectors
|
|
2896
|
-
deleteNestedValue(partitionDoc, fieldPath);
|
|
2934
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
2935
|
+
if (vector !== null || getNestedValue(partitionDoc, fieldPath) == null) deleteNestedValue(partitionDoc, fieldPath);
|
|
2897
2936
|
}
|
|
2898
2937
|
return { partitionDoc };
|
|
2899
2938
|
}
|
|
@@ -2958,7 +2997,7 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
|
|
|
2958
2997
|
if (vecIndexes.size > 0) {
|
|
2959
2998
|
const dimensionCheckVectors = /* @__PURE__ */ new Map();
|
|
2960
2999
|
for (const fieldPath of entry.vectorFieldPaths) {
|
|
2961
|
-
const newVec =
|
|
3000
|
+
const newVec = extractVectorFromDoc(document, fieldPath);
|
|
2962
3001
|
extractedVectors.set(fieldPath, newVec);
|
|
2963
3002
|
if (newVec) {
|
|
2964
3003
|
dimensionCheckVectors.set(fieldPath, newVec);
|
|
@@ -2996,9 +3035,15 @@ function applyOfUpdate(ctx, indexName, prepared) {
|
|
|
2996
3035
|
prepared.buffered = true;
|
|
2997
3036
|
return queued;
|
|
2998
3037
|
}
|
|
3038
|
+
if (manager.has(docId)) {
|
|
3039
|
+
for (const [fieldPath, vector] of extractedVectors) {
|
|
3040
|
+
if (vector !== null) manager.getVectorIndexes().get(fieldPath)?.requireRoomFor(docId, vector);
|
|
3041
|
+
}
|
|
3042
|
+
}
|
|
2999
3043
|
prepared.oldDocument = extractedVectors.size > 0 || ctx.durability !== null || ctx.pluginRegistry.hasHooks("afterUpdate") ? manager.get(docId) : void 0;
|
|
3000
3044
|
prepared.rollbackDoc = prepared.oldDocument;
|
|
3001
3045
|
prepared.rollbackRef = ctx.durability === null ? void 0 : manager.getRef(docId);
|
|
3046
|
+
if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
|
|
3002
3047
|
const stored = manager.update(docId, partitionDoc, {
|
|
3003
3048
|
onRemovedSpellings: (spellings) => {
|
|
3004
3049
|
prepared.removedSpellings = spellings;
|
|
@@ -3060,6 +3105,8 @@ async function updateDocument(ctx, indexName, docId, document, options) {
|
|
|
3060
3105
|
prepared.rollbackRef
|
|
3061
3106
|
);
|
|
3062
3107
|
throw err;
|
|
3108
|
+
} finally {
|
|
3109
|
+
prepared.releaseVectorHolds?.();
|
|
3063
3110
|
}
|
|
3064
3111
|
} else {
|
|
3065
3112
|
await apply();
|
|
@@ -3108,25 +3155,30 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
|
|
|
3108
3155
|
}
|
|
3109
3156
|
}
|
|
3110
3157
|
const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3158
|
+
let failures;
|
|
3159
|
+
try {
|
|
3160
|
+
failures = await recordChunk(ctx, indexName, prepared, applies);
|
|
3161
|
+
for (let i = prepared.length - 1; i >= 0; i--) {
|
|
3162
|
+
const failure = failures[i];
|
|
3163
|
+
if (failure === null) continue;
|
|
3164
|
+
const update = prepared[i];
|
|
3165
|
+
try {
|
|
3166
|
+
await rollbackUpdatedDocument(
|
|
3167
|
+
ctx,
|
|
3168
|
+
indexName,
|
|
3169
|
+
update.docId,
|
|
3170
|
+
update.rollbackDoc,
|
|
3171
|
+
failure.error,
|
|
3172
|
+
update.preparedText,
|
|
3173
|
+
update.appliedRef,
|
|
3174
|
+
update.rollbackRef
|
|
3175
|
+
);
|
|
3176
|
+
} catch (error) {
|
|
3177
|
+
failure.error = error;
|
|
3178
|
+
}
|
|
3129
3179
|
}
|
|
3180
|
+
} finally {
|
|
3181
|
+
for (const update of prepared) update.releaseVectorHolds?.();
|
|
3130
3182
|
}
|
|
3131
3183
|
for (let i = 0; i < prepared.length; i++) {
|
|
3132
3184
|
const update = prepared[i];
|
|
@@ -3193,4 +3245,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
|
|
|
3193
3245
|
core.watermarkNotifier.check(indexName);
|
|
3194
3246
|
}
|
|
3195
3247
|
|
|
3196
|
-
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|
|
3248
|
+
export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
|
|
@@ -560,6 +560,7 @@ function analyseQueryPhrases(parts, analyseText) {
|
|
|
560
560
|
wordsAfterLastPhrase = "";
|
|
561
561
|
if (size > largestPhraseSize) largestPhraseSize = size;
|
|
562
562
|
if (ordered.length > largestTokenCount) largestTokenCount = ordered.length;
|
|
563
|
+
if (size > MAX_PHRASE_WORDS || ordered.length > MAX_PHRASE_WORDS) continue;
|
|
563
564
|
const tokens = ordered.map((entry) => entry.token);
|
|
564
565
|
const offsets = ordered.map((entry) => entry.position - firstPosition);
|
|
565
566
|
const identity = identityOf(tokens, offsets);
|
|
@@ -52,6 +52,14 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
|
|
|
52
52
|
{ expected: embeddingAdapter.dimensions, actual: raw.length }
|
|
53
53
|
);
|
|
54
54
|
}
|
|
55
|
+
for (let index = 0; index < raw.length; index++) {
|
|
56
|
+
if (Number.isFinite(raw[index])) continue;
|
|
57
|
+
throw new NarsilError(
|
|
58
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
59
|
+
`Adapter returned vector with ${raw[index]} at index ${index} for query embedding`,
|
|
60
|
+
{ index }
|
|
61
|
+
);
|
|
62
|
+
}
|
|
55
63
|
const resolved = { ...params.vector, value: Array.from(raw) };
|
|
56
64
|
delete resolved.text;
|
|
57
65
|
return { ...params, vector: resolved };
|