@mcampa/ai-context-core 0.0.2 → 0.1.0-beta.ff0e631
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -47
- package/dist/.tsbuildinfo +1 -1
- package/dist/context.d.ts +30 -5
- package/dist/context.d.ts.map +1 -1
- package/dist/context.js +160 -16
- package/dist/context.js.map +1 -1
- package/dist/embedding/base-embedding.d.ts.map +1 -1
- package/dist/embedding/base-embedding.js +4 -0
- package/dist/embedding/base-embedding.js.map +1 -1
- package/dist/embedding/gemini-embedding.d.ts +43 -1
- package/dist/embedding/gemini-embedding.d.ts.map +1 -1
- package/dist/embedding/gemini-embedding.js +164 -26
- package/dist/embedding/gemini-embedding.js.map +1 -1
- package/dist/embedding/huggingface-embedding.d.ts +70 -0
- package/dist/embedding/huggingface-embedding.d.ts.map +1 -0
- package/dist/embedding/huggingface-embedding.js +270 -0
- package/dist/embedding/huggingface-embedding.js.map +1 -0
- package/dist/embedding/index.d.ts +3 -2
- package/dist/embedding/index.d.ts.map +1 -1
- package/dist/embedding/index.js +3 -2
- package/dist/embedding/index.js.map +1 -1
- package/dist/embedding/ollama-embedding.d.ts +2 -1
- package/dist/embedding/ollama-embedding.d.ts.map +1 -1
- package/dist/embedding/ollama-embedding.js +2 -3
- package/dist/embedding/ollama-embedding.js.map +1 -1
- package/dist/embedding/openai-embedding.d.ts +2 -1
- package/dist/embedding/openai-embedding.d.ts.map +1 -1
- package/dist/embedding/openai-embedding.js +3 -3
- package/dist/embedding/openai-embedding.js.map +1 -1
- package/dist/embedding/voyageai-embedding.d.ts +2 -1
- package/dist/embedding/voyageai-embedding.d.ts.map +1 -1
- package/dist/embedding/voyageai-embedding.js +2 -2
- package/dist/embedding/voyageai-embedding.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -4
- package/dist/index.js.map +1 -1
- package/dist/splitter/ast-splitter.d.ts +1 -1
- package/dist/splitter/ast-splitter.d.ts.map +1 -1
- package/dist/splitter/ast-splitter.js +7 -7
- package/dist/splitter/ast-splitter.js.map +1 -1
- package/dist/splitter/index.d.ts +4 -4
- package/dist/splitter/index.d.ts.map +1 -1
- package/dist/splitter/index.js +1 -1
- package/dist/splitter/index.js.map +1 -1
- package/dist/splitter/langchain-splitter.d.ts +1 -1
- package/dist/splitter/langchain-splitter.d.ts.map +1 -1
- package/dist/splitter/langchain-splitter.js +2 -2
- package/dist/splitter/langchain-splitter.js.map +1 -1
- package/dist/sync/merkle.js +1 -1
- package/dist/sync/merkle.js.map +1 -1
- package/dist/sync/synchronizer.js +6 -6
- package/dist/sync/synchronizer.js.map +1 -1
- package/dist/utils/env-manager.js +4 -4
- package/dist/utils/env-manager.js.map +1 -1
- package/dist/vectordb/base/base-vector-database.d.ts +58 -0
- package/dist/vectordb/base/base-vector-database.d.ts.map +1 -0
- package/dist/vectordb/base/base-vector-database.js +32 -0
- package/dist/vectordb/base/base-vector-database.js.map +1 -0
- package/dist/vectordb/factory.d.ts +113 -0
- package/dist/vectordb/factory.d.ts.map +1 -0
- package/dist/vectordb/factory.js +170 -0
- package/dist/vectordb/factory.js.map +1 -0
- package/dist/vectordb/faiss-vectordb.d.ts +162 -0
- package/dist/vectordb/faiss-vectordb.d.ts.map +1 -0
- package/dist/vectordb/faiss-vectordb.js +777 -0
- package/dist/vectordb/faiss-vectordb.js.map +1 -0
- package/dist/vectordb/index.d.ts +13 -4
- package/dist/vectordb/index.d.ts.map +1 -1
- package/dist/vectordb/index.js +39 -5
- package/dist/vectordb/index.js.map +1 -1
- package/dist/vectordb/libsql-vectordb.d.ts +170 -0
- package/dist/vectordb/libsql-vectordb.d.ts.map +1 -0
- package/dist/vectordb/libsql-vectordb.js +837 -0
- package/dist/vectordb/libsql-vectordb.js.map +1 -0
- package/dist/vectordb/milvus-restful-vectordb.d.ts +12 -11
- package/dist/vectordb/milvus-restful-vectordb.d.ts.map +1 -1
- package/dist/vectordb/milvus-restful-vectordb.js +29 -31
- package/dist/vectordb/milvus-restful-vectordb.js.map +1 -1
- package/dist/vectordb/milvus-vectordb.d.ts +12 -12
- package/dist/vectordb/milvus-vectordb.d.ts.map +1 -1
- package/dist/vectordb/milvus-vectordb.js +31 -28
- package/dist/vectordb/milvus-vectordb.js.map +1 -1
- package/dist/vectordb/qdrant-vectordb.d.ts +149 -0
- package/dist/vectordb/qdrant-vectordb.d.ts.map +1 -0
- package/dist/vectordb/qdrant-vectordb.js +856 -0
- package/dist/vectordb/qdrant-vectordb.js.map +1 -0
- package/dist/vectordb/sparse/index.d.ts +4 -0
- package/dist/vectordb/sparse/index.d.ts.map +1 -0
- package/dist/vectordb/sparse/index.js +23 -0
- package/dist/vectordb/sparse/index.js.map +1 -0
- package/dist/vectordb/sparse/simple-bm25.d.ts +115 -0
- package/dist/vectordb/sparse/simple-bm25.d.ts.map +1 -0
- package/dist/vectordb/sparse/simple-bm25.js +249 -0
- package/dist/vectordb/sparse/simple-bm25.js.map +1 -0
- package/dist/vectordb/sparse/sparse-vector-generator.d.ts +54 -0
- package/dist/vectordb/sparse/sparse-vector-generator.d.ts.map +1 -0
- package/dist/vectordb/sparse/sparse-vector-generator.js +3 -0
- package/dist/vectordb/sparse/sparse-vector-generator.js.map +1 -0
- package/dist/vectordb/sparse/types.d.ts +38 -0
- package/dist/vectordb/sparse/types.d.ts.map +1 -0
- package/dist/vectordb/sparse/types.js +3 -0
- package/dist/vectordb/sparse/types.js.map +1 -0
- package/dist/vectordb/types.d.ts +16 -16
- package/dist/vectordb/types.d.ts.map +1 -1
- package/dist/vectordb/types.js.map +1 -1
- package/dist/vectordb/zilliz-utils.js +3 -3
- package/dist/vectordb/zilliz-utils.js.map +1 -1
- package/package.json +32 -22
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"qdrant-vectordb.js","sourceRoot":"","sources":["../../src/vectordb/qdrant-vectordb.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUA,qDAAuC;AACvC,4CAA8B;AAC9B,gDAAkC;AAClC,2DAAgE;AAChE,sEAAiE;AACjE,sDAAkD;AAqBlD;;;;;;;;;;;;;GAaG;AACH,MAAa,oBAAqB,SAAQ,yCAAgC;IAQxE,YAAY,MAAoB;QAC9B,KAAK,CAAC,MAAM,CAAC,CAAC;QARR,WAAM,GAAwB,IAAI,CAAC;QAG3C,8BAA8B;QACb,sBAAiB,GAAG,OAAO,CAAC;QAC5B,uBAAkB,GAAG,QAAQ,CAAC;QAI7C,IAAI,CAAC,aAAa,GAAG,IAAI,wBAAU,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACO,KAAK,CAAC,UAAU;QACxB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QACpD,MAAM,IAAI,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB,CAAC,OAAe;QAC5C,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,OAAO,CAAC,CAAC;QAE/D,yCAAyC;QACzC,MAAM,GAAG,GAAG,IAAI,GAAG,CACjB,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,OAAO,EAAE,CAC3D,CAAC;QACF,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;QAC1B,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAEzD,IAAI,CAAC,MAAM,GAAG,IAAI,6BAAY,CAAC;YAC7B,IAAI;YACJ,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM;YAC1B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,KAAK;SACtC,CAAC,CAAC;QAEH,4DAA4D;QAC5D,wEAAwE;QACxE,oEAAoE;QACpE,IACE,IAAI,CAAC,MAAM;YACX,OAAQ,IAAI,CAAC,MAAc,CAAC,eAAe,KAAK,UAAU,EAC1D,CAAC;YACA,IAAI,CAAC,MAAc,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,cAAc;QAC5B,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;IAC7B,CAAC;IAED;;OAEG;IACgB,KAAK,CAAC,iBAAiB;QACxC,MAAM,KAAK,CAAC,iBAAiB,EAAE,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED;;;OAGG;IACO,KAAK,CAAC,YAAY,CAAC,eAAuB;QAClD,0DAA0D;QAC1D,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,cAAsB,EACtB,SAAiB,EACjB,YAAqB;QAErB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,oCAAoC,EAAE,cAAc,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,iCAAiC,EAAE,SAAS,CAAC,CAAC;QAE1D,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;YAC3C,cAAc;YACd,aAAa,EAAE;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE;wBACL,GAAG,EAAE;4BACH,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;gCACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC;gCACvB,QAAQ,EAAE,CAAC,EAAE,aAAa;6BAC3B;yBACF;qBACF;iBACF;aACF;SACF,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB,CAC1B,cAAsB,EACtB,SAAiB,EACjB,YAAqB;QAErB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,2CAA2C,EAAE,cAAc,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,uCAAuC,EAAE,SAAS,CAAC,CAAC;QAEhE,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;YAC3C,cAAc;YACd,aAAa,EAAE;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,WAAW;oBACjB,KAAK,EAAE;wBACL,GAAG,EAAE;4BACH,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;gCACxB,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC;gCACvB,QAAQ,EAAE,CAAC,EAAE,aAAa;6BAC3B;yBACF;qBACF;iBACF;aACF;YACD,mBAAmB,EAAE;gBACnB,GAAG,EAAE;oBACH,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;wBACzB,QAAQ,EAAE,yBAAQ,CAAC,GAAG;qBACvB;iBACF;aACF;SACF,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,cAAsB;QACzC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,OAAO,CAAC,GAAG,CAAC,sCAAsC,EAAE,cAAc,CAAC,CAAC;QACpE,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC;YAC3C,cAAc;SACf,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,cAAsB;QACxC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC;gBACzD,cAAc;aACf,CAAC,CAAC;YACH,OAAO,QAAQ,CAAC,MAAM,KAAK,SAAS,CAAC;QACvC,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,+DAA+D;YAC/D,IACE,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,6BAA6B;gBACjD,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,WAAW,CAAC;gBACvC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,gBAAgB,CAAC;gBAC5C,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC;gBACpC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,gBAAgB,CAAC,EACzC,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAE/B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChE,OAAO,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAmB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,cAAsB,EACtB,SAA2B;QAE3B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CACT,yBAAyB,EACzB,SAAS,CAAC,MAAM,EAChB,iBAAiB,EACjB,cAAc,CACf,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACrC,EAAE,EAAE;gBACF,cAAc,EAAE;oBACd,IAAI,EAAE,KAAc;oBACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;iBACvC;aACF;YACD,OAAO,EAAE;gBACP,cAAc,EAAE;oBACd,IAAI,EAAE,SAAkB;oBACxB,KAAK,EAAE;wBACL,OAAO,EAAE;4BACP,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;gCACxB,MAAM,EAAE;oCACN,IAAI,EAAE,OAAgB;oCACtB,KAAK,EAAE;wCACL,IAAI,EAAE,GAAG,CAAC,MAAM;qCACjB;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;YACD,OAAO,EAAE;gBACP,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE;gBACvE,YAAY,EAAE;oBACZ,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,YAAY,EAAE;iBAChE;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,EAAE,IAAI,EAAE,cAAuB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;iBACtE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,EAAE,IAAI,EAAE,cAAuB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;iBACpE;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE;iBACjE;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE;wBACJ,IAAI,EAAE,aAAsB;wBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;qBACpC;iBACF;aACF;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;YACtC,cAAc;YACd,IAAI,EAAE,IAAI;YACV,MAAM;SACP,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;IAC9D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,cAAsB,EACtB,SAA2B;QAE3B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CACT,yBAAyB,EACzB,SAAS,CAAC,MAAM,EAChB,wBAAwB,EACxB,cAAc,CACf,CAAC;QAEF,0CAA0C;QAC1C,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC;YACpC,kGAAkG;YAClG,oFAAoF;YACpF,MAAM,IAAI,KAAK,CACb,0IAA0I,CAC3I,CAAC;QACJ,CAAC;QAED,4CAA4C;QAC5C,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CACzC,CAAC;QAEF,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5C,EAAE,EAAE;gBACF,cAAc,EAAE;oBACd,IAAI,EAAE,KAAc;oBACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,EAAE,CAAC;iBACvC;aACF;YACD,OAAO,EAAE;gBACP,cAAc,EAAE;oBACd,IAAI,EAAE,SAAkB;oBACxB,KAAK,EAAE;wBACL,OAAO,EAAE;4BACP,CAAC,IAAI,CAAC,iBAAiB,CAAC,EAAE;gCACxB,MAAM,EAAE;oCACN,IAAI,EAAE,OAAgB;oCACtB,KAAK,EAAE;wCACL,IAAI,EAAE,GAAG,CAAC,MAAM;qCACjB;iCACF;6BACF;4BACD,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE;gCACzB,MAAM,EAAE;oCACN,IAAI,EAAE,QAAiB;oCACvB,KAAK,EAAE;wCACL,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,OAAO;wCACrC,MAAM,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,MAAM;qCACpC;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;YACD,OAAO,EAAE;gBACP,OAAO,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,EAAE;gBACvE,YAAY,EAAE;oBACZ,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,YAAY,EAAE;iBAChE;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,EAAE,IAAI,EAAE,cAAuB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;iBACtE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,EAAE,IAAI,EAAE,cAAuB,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE;iBACpE;gBACD,aAAa,EAAE;oBACb,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE,GAAG,CAAC,aAAa,EAAE;iBACjE;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE;wBACJ,IAAI,EAAE,aAAsB;wBAC5B,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;qBACpC;iBACF;aACF;SACF,CAAC,CAAC,CAAC;QAEJ,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;YACtC,cAAc;YACd,IAAI,EAAE,IAAI;YACV,MAAM;SACP,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACrE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACV,cAAsB,EACtB,WAAqB,EACrB,OAAuB;QAEvB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CAAC,wCAAwC,EAAE,cAAc,CAAC,CAAC;QAEtE,MAAM,YAAY,GAAQ;YACxB,cAAc;YACd,MAAM,EAAE,WAAW;YACnB,UAAU,EAAE,IAAI,CAAC,iBAAiB;YAClC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAClC,gEAAgE;YAChE,sDAAsD;SACvD,CAAC;QAEF,2BAA2B;QAC3B,IAAI,OAAO,EAAE,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAEtE,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ,EAAE;gBACR,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACtD,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,IAAI,EAAE;gBACnD,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,IAAI,EAAE;gBAC7D,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY,IAAI,CAAC,CAAC;gBAC/D,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,IAAI,CAAC,CAAC;gBAC3D,aAAa,EAAE,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,IAAI,EAAE;gBAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,IAAI,IAAI,CAAC;aACpE;YACD,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAChB,cAAsB,EACtB,cAAqC,EACrC,OAA6B;QAE7B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CACT,uDAAuD,EACvD,cAAc,CACf,CAAC;QAEF,oFAAoF;QACpF,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5E,MAAM,YAAY,GAAG,cAAc,CAAC,IAAI,CACtC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CACtC,CAAC;QAEF,IAAI,CAAC,aAAa,IAAI,CAAC,YAAY,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CACb,qGAAqG,CACtG,CAAC;QACJ,CAAC;QAED,MAAM,UAAU,GAAG,aAAa,CAAC,IAAgB,CAAC;QAClD,MAAM,SAAS,GAAG,YAAY,CAAC,IAAc,CAAC;QAE9C,oCAAoC;QACpC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC;YACpC,OAAO,CAAC,IAAI,CACV,oFAAoF,CACrF,CAAC;QACJ,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE;YAChD,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC;YACxC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QAEhC,OAAO,CAAC,GAAG,CAAC,0CAA0C,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3E,OAAO,CAAC,GAAG,CACT,mCAAmC,EACnC,WAAW,CAAC,OAAO,CAAC,MAAM,CAC3B,CAAC;QACF,OAAO,CAAC,GAAG,CACT,qCAAqC,EACrC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAChC,CAAC;QACF,OAAO,CAAC,GAAG,CACT,oCAAoC,EACpC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAC/B,CAAC;QAEF,uCAAuC;QACvC,IACE,WAAW,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAChC,WAAW,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YAC/B,WAAW,CAAC,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,MAAM,CAAC,MAAM,EACxD,CAAC;YACD,OAAO,CAAC,IAAI,CACV,kFAAkF,CACnF,CAAC;YACF,OAAO,CAAC,IAAI,CACV,iCAAiC,WAAW,CAAC,OAAO,CAAC,MAAM,mBAAmB,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,CAC1G,CAAC;YACF,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,UAAU,EAAE;gBACnD,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;gBAC1B,UAAU,EAAE,OAAO,EAAE,UAAU;aAChC,CAAC,CAAC;QACL,CAAC;QAED,2EAA2E;QAC3E,MAAM,iBAAiB,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,IAAI,iBAAiB,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,iFAAiF,CAClF,CAAC;YACF,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;YACjE,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,UAAU,EAAE;gBACnD,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;gBAC1B,UAAU,EAAE,OAAO,EAAE,UAAU;aAChC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,GAAG,CACT,oEAAoE,CACrE,CAAC;QAEF,0DAA0D;QAC1D,yEAAyE;QACzE,2FAA2F;QAC3F,EAAE;QACF,0EAA0E;QAC1E,yDAAyD;QACzD,EAAE;QACF,yBAAyB;QACzB,6BAA6B;QAC7B,gCAAgC;QAChC,oBAAoB;QACpB,EAAE;QACF,2BAA2B;QAC3B,sDAAsD;QACtD,mDAAmD;QACnD,gDAAgD;QAChD,qBAAqB;QACrB,EAAE;QACF,mCAAmC;QACnC,iGAAiG;QACjG,EAAE;QACF,yCAAyC;QACzC,qGAAqG;QACrG,EAAE;QACF,kCAAkC;QAClC,wDAAwD;QACxD,iCAAiC;QAEjC,MAAM,WAAW,GAAQ;YACvB,cAAc;YACd,QAAQ,EAAE;gBACR;oBACE,wBAAwB;oBACxB,KAAK,EAAE;wBACL,OAAO,EAAE;4BACP,IAAI,EAAE,SAAkB;4BACxB,KAAK,EAAE;gCACL,OAAO,EAAE;oCACP,IAAI,EAAE,OAAgB;oCACtB,KAAK,EAAE;wCACL,IAAI,EAAE,UAAU;qCACjB;iCACF;6BACF;yBACF;qBACF;oBACD,KAAK,EAAE,IAAI,CAAC,iBAAiB;oBAC7B,KAAK,EAAE,MAAM,CAAC,aAAa,CAAC,KAAK,IAAI,EAAE,CAAC;iBACzC;gBACD;oBACE,yBAAyB;oBACzB,KAAK,EAAE;wBACL,OAAO,EAAE;4BACP,IAAI,EAAE,SAAkB;4BACxB,KAAK,EAAE;gCACL,OAAO,EAAE;oCACP,IAAI,EAAE,QAAiB;oCACvB,KAAK,EAAE;wCACL,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wCAClD,MAAM,EAAE,WAAW,CAAC,MAAM;qCAC3B;iCACF;6BACF;yBACF;qBACF;oBACD,KAAK,EAAE,IAAI,CAAC,kBAAkB;oBAC9B,KAAK,EAAE,MAAM,CAAC,YAAY,CAAC,KAAK,IAAI,EAAE,CAAC;iBACxC;aACF;YACD,kDAAkD;YAClD,KAAK,EAAE;gBACL,OAAO,EAAE;oBACP,IAAI,EAAE,QAAiB;oBACvB,KAAK,EAAE,CAAU,EAAE,iBAAiB;iBACrC;aACF;YACD,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;YACnC,WAAW,EAAE;gBACX,eAAe,EAAE;oBACf,IAAI,EAAE,QAAiB;oBACvB,KAAK,EAAE,IAAI;iBACZ;aACF;SACF,CAAC;QAEF,2BAA2B;QAC3B,IAAI,OAAO,EAAE,UAAU,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChE,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAEpE,OAAO,CAAC,GAAG,CACT,oBAAoB,EACpB,OAAO,CAAC,MAAM,CAAC,MAAM,EACrB,4BAA4B,CAC7B,CAAC;QAEF,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAW,EAAE,EAAE,CAAC,CAAC;YAC1C,QAAQ,EAAE;gBACR,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE;gBACtD,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE;gBACnD,MAAM,EAAE,EAAE;gBACV,YAAY,EAAE,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE;gBAC7D,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC;gBAC9D,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC;gBAC1D,aAAa,EAAE,MAAM,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE;gBAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;aACpE;YACD,KAAK,EAAE,MAAM,CAAC,KAAK;SACpB,CAAC,CAAC,CAAC;IACN,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,cAAsB,EAAE,GAAa;QAChD,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CACT,0BAA0B,EAC1B,GAAG,CAAC,MAAM,EACV,iBAAiB,EACjB,cAAc,CACf,CAAC;QAEF,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;YACtC,cAAc;YACd,IAAI,EAAE,IAAI;YACV,MAAM,EAAE;gBACN,mBAAmB,EAAE;oBACnB,IAAI,EAAE,QAAiB;oBACvB,KAAK,EAAE;wBACL,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;4BACpB,cAAc,EAAE;gCACd,IAAI,EAAE,KAAc;gCACpB,KAAK,EAAE,IAAI,CAAC,kBAAkB,CAAC,EAAE,CAAC;6BACnC;yBACF,CAAC,CAAC;qBACJ;iBACF;aACF;SACF,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,cAAsB,EACtB,MAAc,EACd,YAAsB,EACtB,KAAc;QAEd,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,MAAM,IAAI,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC;QAExC,OAAO,CAAC,GAAG,CAAC,oCAAoC,EAAE,cAAc,CAAC,CAAC;QAElE,0BAA0B;QAC1B,gEAAgE;QAChE,4DAA4D;QAC5D,MAAM,YAAY,GAAQ;YACxB,cAAc;YACd,KAAK,EAAE,KAAK,IAAI,GAAG;YACnB,UAAU,EAAE,KAAK;SAClB,CAAC;QAEF,sCAAsC;QACtC,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvC,YAAY,CAAC,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAEtE,0DAA0D;QAC1D,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YACvC,qCAAqC;YACrC,yGAAyG;YACzG,IAAI,OAAO,GAAG,EAAE,CAAC;YACjB,IAAI,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC;gBAC7C,OAAO,GAAG,KAAK,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC;gBACpD,OAAO,GAAG,KAAK,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC;YAC1C,CAAC;iBAAM,IAAI,KAAK,CAAC,EAAE,EAAE,GAAG,KAAK,SAAS,EAAE,CAAC;gBACvC,sCAAsC;gBACtC,OAAO,GAAG,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;YACpC,CAAC;iBAAM,IAAI,KAAK,CAAC,EAAE,EAAE,GAAG,KAAK,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC;YACzB,CAAC;YAED,MAAM,MAAM,GAAwB;gBAClC,EAAE,EAAE,OAAO;aACZ,CAAC;YAEF,2DAA2D;YAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,4FAA4F;gBAC5F,MAAM,CAAC,OAAO;oBACZ,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK;wBACnC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,WAAW,CAAC;gBACtC,MAAM,CAAC,YAAY;oBACjB,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK;wBACxC,KAAK,CAAC,OAAO,EAAE,YAAY,EAAE,WAAW,CAAC;gBAC3C,MAAM,CAAC,SAAS,GAAG,MAAM,CACvB,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK;oBACnC,KAAK,CAAC,OAAO,EAAE,SAAS,EAAE,YAAY;oBACtC,CAAC,CACJ,CAAC;gBACF,MAAM,CAAC,OAAO,GAAG,MAAM,CACrB,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK;oBACjC,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY;oBACpC,CAAC,CACJ,CAAC;gBACF,MAAM,CAAC,aAAa;oBAClB,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,KAAK;wBACzC,KAAK,CAAC,OAAO,EAAE,aAAa,EAAE,WAAW,CAAC;gBAC5C,MAAM,WAAW,GACf,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK;oBACpC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC;gBACvC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,gCAAgC;gBAChC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;oBACjC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACnC,2DAA2D;wBAC3D,kFAAkF;wBAClF,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;4BACvC,MAAM,CAAC,KAAK,CAAC;gCACX,KAAK,KAAK,UAAU;oCAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;oCACtC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;wBACzB,CAAC;6BAAM,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,cAAc,EAAE,CAAC;4BAC/C,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAC3C,CAAC;6BAAM,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,aAAa,EAAE,CAAC;4BAC9C,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;wBACnC,CAAC;6BAAM,IAAI,KAAK,CAAC,IAAI,EAAE,IAAI,KAAK,WAAW,EAAE,CAAC;4BAC5C,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;wBACnC,CAAC;wBACD,4DAA4D;6BACvD,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;4BACzC,MAAM,CAAC,KAAK,CAAC;gCACX,KAAK,KAAK,UAAU;oCAClB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC;oCACvC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC;wBAC1B,CAAC;6BAAM,IAAI,KAAK,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;4BAC5C,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;wBAC7C,CAAC;6BAAM,IAAI,KAAK,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;4BAC3C,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC;wBACpC,CAAC;6BAAM,IAAI,KAAK,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;4BACzC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC;wBAClC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB;QACxB,oEAAoE;QACpE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;OAMG;IACK,qBAAqB,CAAC,IAAY;QACxC,2CAA2C;QAC3C,4DAA4D;QAE5D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,kCAAkC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAClD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;qBACpB,KAAK,CAAC,GAAG,CAAC;qBACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;gBAE7C,iFAAiF;gBACjF,OAAO;oBACL,IAAI,EAAE;wBACJ;4BACE,cAAc,EAAE;gCACd,IAAI,EAAE,OAAgB;gCACtB,KAAK,EAAE;oCACL,GAAG,EAAE,KAAK;oCACV,KAAK,EAAE;wCACL,UAAU,EAAE;4CACV,IAAI,EAAE,KAAc;4CACpB,KAAK,EAAE;gDACL,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oDAC7B,IAAI,EAAE,EAAE,IAAI,EAAE,aAAsB,EAAE,KAAK,EAAE;iDAC9C,CAAC,CAAC;6CACJ;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,kCAAkC;YAClC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACvB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE9B,OAAO;oBACL,IAAI,EAAE;wBACJ;4BACE,cAAc,EAAE;gCACd,IAAI,EAAE,OAAgB;gCACtB,KAAK,EAAE;oCACL,GAAG,EAAE,KAAK;oCACV,KAAK,EAAE;wCACL,UAAU,EAAE;4CACV,IAAI,EAAE,SAAkB;4CACxB,KAAK;yCACN;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,oDAAoD;QACpD,OAAO,CAAC,IAAI,CAAC,mDAAmD,EAAE,IAAI,CAAC,CAAC;QACxE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,OAAe;QACxC,yDAAyD;QACzD,MAAM,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC1C,wDAAwD;QACxD,OAAO,MAAM,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACI,gBAAgB;QACrB,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,cAAsB;QAC7C,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,cAAc,OAAO,CAAC,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,cAAsB;QACxC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,yDAAyD,CAAC,CAAC;YACvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAEzC,0BAA0B;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAE9C,gCAAgC;YAChC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC9C,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;YAElD,OAAO,CAAC,GAAG,CAAC,sCAAsC,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,aAAa,CAAC,cAAsB;QACxC,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YAExD,6BAA6B;YAC7B,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CACT,gDAAgD,SAAS,EAAE,CAC5D,CAAC;gBACF,OAAO,KAAK,CAAC;YACf,CAAC;YAED,kCAAkC;YAClC,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACxD,IAAI,CAAC,aAAa,GAAG,wBAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YAEpD,OAAO,CAAC,GAAG,CAAC,yCAAyC,SAAS,EAAE,CAAC,CAAC;YAClE,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;YAChE,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe,CAAC,cAAsB;QAC1C,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YAExD,6BAA6B;YAC7B,IAAI,CAAC;gBACH,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC7B,CAAC;YAAC,MAAM,CAAC;gBACP,wCAAwC;gBACxC,OAAO;YACT,CAAC;YAED,MAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC3B,OAAO,CAAC,GAAG,CAAC,0CAA0C,SAAS,EAAE,CAAC,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;CACF;AAx8BD,oDAw8BC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/index.ts"],"names":[],"mappings":"AACA,cAAc,eAAe,CAAC;AAG9B,cAAc,2BAA2B,CAAC;AAG1C,cAAc,SAAS,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
// Implementations
|
|
18
|
+
__exportStar(require("./simple-bm25"), exports);
|
|
19
|
+
// Interface
|
|
20
|
+
__exportStar(require("./sparse-vector-generator"), exports);
|
|
21
|
+
// Types
|
|
22
|
+
__exportStar(require("./types"), exports);
|
|
23
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kBAAkB;AAClB,gDAA8B;AAE9B,YAAY;AACZ,4DAA0C;AAE1C,QAAQ;AACR,0CAAwB"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import type { SparseVectorGenerator } from "./sparse-vector-generator";
|
|
2
|
+
import type { SparseVector, SparseVectorConfig } from "./types";
|
|
3
|
+
/**
|
|
4
|
+
* Configuration for BM25 algorithm
|
|
5
|
+
*/
|
|
6
|
+
export interface BM25Config {
|
|
7
|
+
/**
|
|
8
|
+
* Term frequency saturation parameter
|
|
9
|
+
* Controls how quickly term frequency impact saturates
|
|
10
|
+
* @default 1.2
|
|
11
|
+
*/
|
|
12
|
+
k1?: number;
|
|
13
|
+
/**
|
|
14
|
+
* Length normalization parameter
|
|
15
|
+
* 0 = no normalization, 1 = full normalization
|
|
16
|
+
* @default 0.75
|
|
17
|
+
*/
|
|
18
|
+
b?: number;
|
|
19
|
+
/**
|
|
20
|
+
* Minimum term length to include
|
|
21
|
+
* @default 2
|
|
22
|
+
*/
|
|
23
|
+
minTermLength?: number;
|
|
24
|
+
/**
|
|
25
|
+
* Stop words to exclude
|
|
26
|
+
* @default []
|
|
27
|
+
*/
|
|
28
|
+
stopWords?: Set<string>;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Simple BM25 implementation for sparse vector generation
|
|
32
|
+
*
|
|
33
|
+
* BM25 (Best Matching 25) is a probabilistic ranking function
|
|
34
|
+
* used for information retrieval and text search.
|
|
35
|
+
*
|
|
36
|
+
* Formula:
|
|
37
|
+
* score(D,Q) = Σ IDF(qi) × (f(qi,D) × (k1 + 1)) / (f(qi,D) + k1 × (1 - b + b × |D| / avgdl))
|
|
38
|
+
*
|
|
39
|
+
* Where:
|
|
40
|
+
* - D = document
|
|
41
|
+
* - Q = query
|
|
42
|
+
* - qi = query term i
|
|
43
|
+
* - f(qi,D) = term frequency in document
|
|
44
|
+
* - |D| = document length
|
|
45
|
+
* - avgdl = average document length
|
|
46
|
+
* - IDF(qi) = inverse document frequency
|
|
47
|
+
*/
|
|
48
|
+
export declare class SimpleBM25 implements SparseVectorGenerator {
|
|
49
|
+
private k1;
|
|
50
|
+
private b;
|
|
51
|
+
private minTermLength;
|
|
52
|
+
private stopWords;
|
|
53
|
+
private vocabulary;
|
|
54
|
+
private idf;
|
|
55
|
+
private avgDocLength;
|
|
56
|
+
private trained;
|
|
57
|
+
constructor(config?: BM25Config);
|
|
58
|
+
/**
|
|
59
|
+
* Tokenize text into terms
|
|
60
|
+
* Simple but effective tokenization:
|
|
61
|
+
* 1. Lowercase
|
|
62
|
+
* 2. Remove special characters
|
|
63
|
+
* 3. Split on whitespace
|
|
64
|
+
* 4. Filter by length and stop words
|
|
65
|
+
*/
|
|
66
|
+
private tokenize;
|
|
67
|
+
/**
|
|
68
|
+
* Calculate term frequency for a document
|
|
69
|
+
*/
|
|
70
|
+
private calculateTermFrequency;
|
|
71
|
+
/**
|
|
72
|
+
* Learn vocabulary and IDF from corpus
|
|
73
|
+
*/
|
|
74
|
+
learn(documents: string[]): void;
|
|
75
|
+
/**
|
|
76
|
+
* Generate sparse vector for a single text
|
|
77
|
+
*/
|
|
78
|
+
generate(text: string, config?: SparseVectorConfig): SparseVector;
|
|
79
|
+
/**
|
|
80
|
+
* Generate sparse vectors for multiple texts (batch operation)
|
|
81
|
+
*/
|
|
82
|
+
generateBatch(texts: string[], config?: SparseVectorConfig): SparseVector[];
|
|
83
|
+
/**
|
|
84
|
+
* Get vocabulary size
|
|
85
|
+
*/
|
|
86
|
+
getVocabularySize(): number;
|
|
87
|
+
/**
|
|
88
|
+
* Get average document length
|
|
89
|
+
*/
|
|
90
|
+
getAverageDocumentLength(): number;
|
|
91
|
+
/**
|
|
92
|
+
* Check if trained
|
|
93
|
+
*/
|
|
94
|
+
isTrained(): boolean;
|
|
95
|
+
/**
|
|
96
|
+
* Get the vocabulary (for debugging/inspection)
|
|
97
|
+
*/
|
|
98
|
+
getVocabulary(): Map<string, number>;
|
|
99
|
+
/**
|
|
100
|
+
* Get IDF scores (for debugging/inspection)
|
|
101
|
+
*/
|
|
102
|
+
getIDFScores(): Map<string, number>;
|
|
103
|
+
/**
|
|
104
|
+
* Serialize the BM25 model to JSON
|
|
105
|
+
* Exports the trained state including vocabulary, IDF scores, and avgDocLength
|
|
106
|
+
* Can serialize untrained models (for empty hybrid collections)
|
|
107
|
+
*/
|
|
108
|
+
toJSON(): string;
|
|
109
|
+
/**
|
|
110
|
+
* Deserialize and load a BM25 model from JSON
|
|
111
|
+
* Restores the trained state including vocabulary, IDF scores, and avgDocLength
|
|
112
|
+
*/
|
|
113
|
+
static fromJSON(json: string): SimpleBM25;
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=simple-bm25.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"simple-bm25.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/simple-bm25.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AACvE,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEhE;;GAEG;AACH,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ;;;;OAIG;IACH,CAAC,CAAC,EAAE,MAAM,CAAC;IAEX;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,SAAS,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,UAAW,YAAW,qBAAqB;IACtD,OAAO,CAAC,EAAE,CAAS;IACnB,OAAO,CAAC,CAAC,CAAS;IAClB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAc;IAG/B,OAAO,CAAC,UAAU,CAAkC;IACpD,OAAO,CAAC,GAAG,CAAkC;IAC7C,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,OAAO,CAAkB;gBAErB,MAAM,GAAE,UAAe;IAOnC;;;;;;;OAOG;IACH,OAAO,CAAC,QAAQ;IAWhB;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAQ9B;;OAEG;IACH,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI;IAmDhC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY;IA2GjE;;OAEG;IACH,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,CAAC,EAAE,kBAAkB,GAAG,YAAY,EAAE;IAI3E;;OAEG;IACH,iBAAiB,IAAI,MAAM;IAI3B;;OAEG;IACH,wBAAwB,IAAI,MAAM;IAIlC;;OAEG;IACH,SAAS,IAAI,OAAO;IAIpB;;OAEG;IACH,aAAa,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAIpC;;OAEG;IACH,YAAY,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAInC;;;;OAIG;IACH,MAAM,IAAI,MAAM;IAahB;;;OAGG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU;CAqB1C"}
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SimpleBM25 = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Simple BM25 implementation for sparse vector generation
|
|
6
|
+
*
|
|
7
|
+
* BM25 (Best Matching 25) is a probabilistic ranking function
|
|
8
|
+
* used for information retrieval and text search.
|
|
9
|
+
*
|
|
10
|
+
* Formula:
|
|
11
|
+
* score(D,Q) = Σ IDF(qi) × (f(qi,D) × (k1 + 1)) / (f(qi,D) + k1 × (1 - b + b × |D| / avgdl))
|
|
12
|
+
*
|
|
13
|
+
* Where:
|
|
14
|
+
* - D = document
|
|
15
|
+
* - Q = query
|
|
16
|
+
* - qi = query term i
|
|
17
|
+
* - f(qi,D) = term frequency in document
|
|
18
|
+
* - |D| = document length
|
|
19
|
+
* - avgdl = average document length
|
|
20
|
+
* - IDF(qi) = inverse document frequency
|
|
21
|
+
*/
|
|
22
|
+
class SimpleBM25 {
|
|
23
|
+
constructor(config = {}) {
|
|
24
|
+
// Learned from corpus
|
|
25
|
+
this.vocabulary = new Map();
|
|
26
|
+
this.idf = new Map();
|
|
27
|
+
this.avgDocLength = 0;
|
|
28
|
+
this.trained = false;
|
|
29
|
+
this.k1 = config.k1 ?? 1.2;
|
|
30
|
+
this.b = config.b ?? 0.75;
|
|
31
|
+
this.minTermLength = config.minTermLength ?? 2;
|
|
32
|
+
this.stopWords = config.stopWords ?? new Set();
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Tokenize text into terms
|
|
36
|
+
* Simple but effective tokenization:
|
|
37
|
+
* 1. Lowercase
|
|
38
|
+
* 2. Remove special characters
|
|
39
|
+
* 3. Split on whitespace
|
|
40
|
+
* 4. Filter by length and stop words
|
|
41
|
+
*/
|
|
42
|
+
tokenize(text) {
|
|
43
|
+
return text
|
|
44
|
+
.toLowerCase()
|
|
45
|
+
.replace(/[^\w\s]/g, " ")
|
|
46
|
+
.split(/\s+/)
|
|
47
|
+
.filter((term) => term.length >= this.minTermLength && !this.stopWords.has(term));
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Calculate term frequency for a document
|
|
51
|
+
*/
|
|
52
|
+
calculateTermFrequency(tokens) {
|
|
53
|
+
const termFreq = new Map();
|
|
54
|
+
tokens.forEach((token) => {
|
|
55
|
+
termFreq.set(token, (termFreq.get(token) || 0) + 1);
|
|
56
|
+
});
|
|
57
|
+
return termFreq;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Learn vocabulary and IDF from corpus
|
|
61
|
+
*/
|
|
62
|
+
learn(documents) {
|
|
63
|
+
if (documents.length === 0) {
|
|
64
|
+
throw new Error("Cannot learn from empty corpus");
|
|
65
|
+
}
|
|
66
|
+
// Reset state
|
|
67
|
+
this.vocabulary.clear();
|
|
68
|
+
this.idf.clear();
|
|
69
|
+
// Tokenize all documents
|
|
70
|
+
const tokenizedDocs = documents.map((doc) => this.tokenize(doc));
|
|
71
|
+
// Calculate average document length
|
|
72
|
+
const totalLength = tokenizedDocs.reduce((sum, tokens) => sum + tokens.length, 0);
|
|
73
|
+
this.avgDocLength = totalLength / tokenizedDocs.length;
|
|
74
|
+
// Calculate document frequency for each term
|
|
75
|
+
const docFreq = new Map();
|
|
76
|
+
tokenizedDocs.forEach((tokens) => {
|
|
77
|
+
const uniqueTerms = new Set(tokens);
|
|
78
|
+
uniqueTerms.forEach((term) => {
|
|
79
|
+
docFreq.set(term, (docFreq.get(term) || 0) + 1);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
// Build vocabulary and calculate IDF
|
|
83
|
+
// IDF formula: log((N - df + 0.5) / (df + 0.5))
|
|
84
|
+
const numDocs = documents.length;
|
|
85
|
+
let vocabIndex = 0;
|
|
86
|
+
docFreq.forEach((df, term) => {
|
|
87
|
+
// Add to vocabulary
|
|
88
|
+
this.vocabulary.set(term, vocabIndex++);
|
|
89
|
+
// Calculate IDF
|
|
90
|
+
const idf = Math.log((numDocs - df + 0.5) / (df + 0.5));
|
|
91
|
+
this.idf.set(term, idf);
|
|
92
|
+
});
|
|
93
|
+
this.trained = true;
|
|
94
|
+
console.log(`[SimpleBM25] Learned from ${numDocs} documents`);
|
|
95
|
+
console.log(`[SimpleBM25] Vocabulary size: ${this.vocabulary.size}`);
|
|
96
|
+
console.log(`[SimpleBM25] Average document length: ${this.avgDocLength.toFixed(2)}`);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Generate sparse vector for a single text
|
|
100
|
+
*/
|
|
101
|
+
generate(text, config) {
|
|
102
|
+
if (!this.trained) {
|
|
103
|
+
throw new Error("BM25 generator must be trained before generating vectors. Call learn() first.");
|
|
104
|
+
}
|
|
105
|
+
const tokens = this.tokenize(text);
|
|
106
|
+
const termFreq = this.calculateTermFrequency(tokens);
|
|
107
|
+
const docLength = tokens.length;
|
|
108
|
+
console.log(`[SimpleBM25] Generate: tokenized "${text.substring(0, 50)}..." into ${tokens.length} tokens`);
|
|
109
|
+
console.log(`[SimpleBM25] Generate: unique terms: ${termFreq.size}, vocabulary size: ${this.vocabulary.size}`);
|
|
110
|
+
const indices = [];
|
|
111
|
+
const values = [];
|
|
112
|
+
let skippedTerms = 0;
|
|
113
|
+
// Calculate BM25 score for each term
|
|
114
|
+
termFreq.forEach((tf, term) => {
|
|
115
|
+
const vocabIndex = this.vocabulary.get(term);
|
|
116
|
+
const idfScore = this.idf.get(term);
|
|
117
|
+
// Skip terms not in vocabulary (unknown terms)
|
|
118
|
+
if (vocabIndex === undefined || idfScore === undefined) {
|
|
119
|
+
skippedTerms++;
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
// BM25 formula
|
|
123
|
+
const normalizedTF = (tf * (this.k1 + 1)) /
|
|
124
|
+
(tf +
|
|
125
|
+
this.k1 * (1 - this.b + (this.b * docLength) / this.avgDocLength));
|
|
126
|
+
const score = idfScore * normalizedTF;
|
|
127
|
+
// Apply minimum score threshold if configured
|
|
128
|
+
if (config?.minScore !== undefined && score < config.minScore) {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
indices.push(vocabIndex);
|
|
132
|
+
values.push(score);
|
|
133
|
+
});
|
|
134
|
+
console.log(`[SimpleBM25] Generate: matched ${indices.length} terms, skipped ${skippedTerms} unknown terms`);
|
|
135
|
+
if (indices.length === 0 && termFreq.size > 0) {
|
|
136
|
+
console.warn(`[SimpleBM25] ⚠️ WARNING: No terms matched in vocabulary! All ${termFreq.size} unique terms were unknown.`);
|
|
137
|
+
console.warn(`[SimpleBM25] ⚠️ Sample query terms: ${Array.from(termFreq.keys()).slice(0, 5).join(", ")}`);
|
|
138
|
+
console.warn(`[SimpleBM25] ⚠️ Sample vocab terms: ${Array.from(this.vocabulary.keys()).slice(0, 5).join(", ")}`);
|
|
139
|
+
}
|
|
140
|
+
// Sort by score descending and apply maxTerms limit
|
|
141
|
+
if (config?.maxTerms !== undefined && indices.length > config.maxTerms) {
|
|
142
|
+
const combined = indices.map((idx, i) => ({ idx, val: values[i] }));
|
|
143
|
+
combined.sort((a, b) => b.val - a.val);
|
|
144
|
+
combined.splice(config.maxTerms);
|
|
145
|
+
indices.length = 0;
|
|
146
|
+
values.length = 0;
|
|
147
|
+
combined.forEach(({ idx, val }) => {
|
|
148
|
+
indices.push(idx);
|
|
149
|
+
values.push(val);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
// Ensure all values are positive (Qdrant requirement for sparse vectors)
|
|
153
|
+
// If there are negative values, shift all values so minimum becomes a small positive
|
|
154
|
+
if (values.length > 0) {
|
|
155
|
+
const minValue = Math.min(...values);
|
|
156
|
+
if (minValue <= 0) {
|
|
157
|
+
// Shift so minimum becomes epsilon (not 0, to preserve all terms)
|
|
158
|
+
const epsilon = 1e-6;
|
|
159
|
+
const shift = -minValue + epsilon;
|
|
160
|
+
console.log(`[SimpleBM25] Shifting values by ${shift} to ensure positivity`);
|
|
161
|
+
for (let i = 0; i < values.length; i++) {
|
|
162
|
+
values[i] += shift;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Normalize if requested
|
|
167
|
+
if (config?.normalize && values.length > 0) {
|
|
168
|
+
const norm = Math.sqrt(values.reduce((sum, val) => sum + val * val, 0));
|
|
169
|
+
for (let i = 0; i < values.length; i++) {
|
|
170
|
+
values[i] /= norm;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return { indices, values };
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Generate sparse vectors for multiple texts (batch operation)
|
|
177
|
+
*/
|
|
178
|
+
generateBatch(texts, config) {
|
|
179
|
+
return texts.map((text) => this.generate(text, config));
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Get vocabulary size
|
|
183
|
+
*/
|
|
184
|
+
getVocabularySize() {
|
|
185
|
+
return this.vocabulary.size;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Get average document length
|
|
189
|
+
*/
|
|
190
|
+
getAverageDocumentLength() {
|
|
191
|
+
return this.avgDocLength;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Check if trained
|
|
195
|
+
*/
|
|
196
|
+
isTrained() {
|
|
197
|
+
return this.trained;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Get the vocabulary (for debugging/inspection)
|
|
201
|
+
*/
|
|
202
|
+
getVocabulary() {
|
|
203
|
+
return new Map(this.vocabulary);
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Get IDF scores (for debugging/inspection)
|
|
207
|
+
*/
|
|
208
|
+
getIDFScores() {
|
|
209
|
+
return new Map(this.idf);
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Serialize the BM25 model to JSON
|
|
213
|
+
* Exports the trained state including vocabulary, IDF scores, and avgDocLength
|
|
214
|
+
* Can serialize untrained models (for empty hybrid collections)
|
|
215
|
+
*/
|
|
216
|
+
toJSON() {
|
|
217
|
+
return JSON.stringify({
|
|
218
|
+
k1: this.k1,
|
|
219
|
+
b: this.b,
|
|
220
|
+
minTermLength: this.minTermLength,
|
|
221
|
+
stopWords: Array.from(this.stopWords),
|
|
222
|
+
vocabulary: Array.from(this.vocabulary.entries()),
|
|
223
|
+
idf: Array.from(this.idf.entries()),
|
|
224
|
+
avgDocLength: this.avgDocLength,
|
|
225
|
+
trained: this.trained,
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Deserialize and load a BM25 model from JSON
|
|
230
|
+
* Restores the trained state including vocabulary, IDF scores, and avgDocLength
|
|
231
|
+
*/
|
|
232
|
+
static fromJSON(json) {
|
|
233
|
+
const data = JSON.parse(json);
|
|
234
|
+
const bm25 = new SimpleBM25({
|
|
235
|
+
k1: data.k1,
|
|
236
|
+
b: data.b,
|
|
237
|
+
minTermLength: data.minTermLength,
|
|
238
|
+
stopWords: new Set(data.stopWords || []),
|
|
239
|
+
});
|
|
240
|
+
bm25.vocabulary = new Map(data.vocabulary);
|
|
241
|
+
bm25.idf = new Map(data.idf);
|
|
242
|
+
bm25.avgDocLength = data.avgDocLength;
|
|
243
|
+
bm25.trained = data.trained;
|
|
244
|
+
console.log(`[SimpleBM25] Loaded model with ${bm25.vocabulary.size} terms, avgDocLength: ${bm25.avgDocLength.toFixed(2)}`);
|
|
245
|
+
return bm25;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
exports.SimpleBM25 = SimpleBM25;
|
|
249
|
+
//# sourceMappingURL=simple-bm25.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"simple-bm25.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/simple-bm25.ts"],"names":[],"mappings":";;;AAkCA;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAa,UAAU;IAYrB,YAAY,SAAqB,EAAE;QANnC,sBAAsB;QACd,eAAU,GAAwB,IAAI,GAAG,EAAE,CAAC;QAC5C,QAAG,GAAwB,IAAI,GAAG,EAAE,CAAC;QACrC,iBAAY,GAAW,CAAC,CAAC;QACzB,YAAO,GAAY,KAAK,CAAC;QAG/B,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,GAAG,CAAC;QAC3B,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,IAAI,GAAG,EAAE,CAAC;IACjD,CAAC;IAED;;;;;;;OAOG;IACK,QAAQ,CAAC,IAAY;QAC3B,OAAO,IAAI;aACR,WAAW,EAAE;aACb,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,KAAK,CAAC,KAAK,CAAC;aACZ,MAAM,CACL,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CACjE,CAAC;IACN,CAAC;IAED;;OAEG;IACK,sBAAsB,CAAC,MAAgB;QAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC3C,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;YACvB,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QACH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAmB;QACvB,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACpD,CAAC;QAED,cAAc;QACd,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACxB,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QAEjB,yBAAyB;QACzB,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAEjE,oCAAoC;QACpC,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,EACpC,CAAC,CACF,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC;QAEvD,6CAA6C;QAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,aAAa,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;YAC/B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;YACpC,WAAW,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBAC3B,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,qCAAqC;QACrC,gDAAgD;QAChD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC3B,oBAAoB;YACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAExC,gBAAgB;YAChB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,OAAO,CAAC,GAAG,CAAC,6BAA6B,OAAO,YAAY,CAAC,CAAC;QAC9D,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,GAAG,CACT,yCAAyC,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACxE,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,QAAQ,CAAC,IAAY,EAAE,MAA2B;QAChD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAEhC,OAAO,CAAC,GAAG,CACT,qCAAqC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,MAAM,CAAC,MAAM,SAAS,CAC9F,CAAC;QACF,OAAO,CAAC,GAAG,CACT,wCAAwC,QAAQ,CAAC,IAAI,sBAAsB,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAClG,CAAC;QAEF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,qCAAqC;QACrC,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE;YAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAEpC,+CAA+C;YAC/C,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACvD,YAAY,EAAE,CAAC;gBACf,OAAO;YACT,CAAC;YAED,eAAe;YACf,MAAM,YAAY,GAChB,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACpB,CAAC,EAAE;oBACD,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YAEvE,MAAM,KAAK,GAAG,QAAQ,GAAG,YAAY,CAAC;YAEtC,8CAA8C;YAC9C,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;gBAC9D,OAAO;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CACT,kCAAkC,OAAO,CAAC,MAAM,mBAAmB,YAAY,gBAAgB,CAChG,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,IAAI,CACV,iEAAiE,QAAQ,CAAC,IAAI,6BAA6B,CAC5G,CAAC;YACF,OAAO,CAAC,IAAI,CACV,wCAAwC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7F,CAAC;YACF,OAAO,CAAC,IAAI,CACV,wCAAwC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpG,CAAC;QACJ,CAAC;QAED,oDAAoD;QACpD,IAAI,MAAM,EAAE,QAAQ,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvE,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YACvC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEjC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACnB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAClB,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE;gBAChC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,yEAAyE;QACzE,qFAAqF;QACrF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YACrC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;gBAClB,kEAAkE;gBAClE,MAAM,OAAO,GAAG,IAAI,CAAC;gBACrB,MAAM,KAAK,GAAG,CAAC,QAAQ,GAAG,OAAO,CAAC;gBAClC,OAAO,CAAC,GAAG,CACT,mCAAmC,KAAK,uBAAuB,CAChE,CAAC;gBACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,yBAAyB;QACzB,IAAI,MAAM,EAAE,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACxE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,aAAa,CAAC,KAAe,EAAE,MAA2B;QACxD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,wBAAwB;QACtB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED;;;;OAIG;IACH,MAAM;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YACrC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;YACjD,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACnC,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;IACL,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,QAAQ,CAAC,IAAY;QAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE9B,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC;YAC1B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,CAAC,EAAE,IAAI,CAAC,CAAC;YACT,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,SAAS,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;SACzC,CAAC,CAAC;QAEH,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QACtC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAE5B,OAAO,CAAC,GAAG,CACT,kCAAkC,IAAI,CAAC,UAAU,CAAC,IAAI,yBAAyB,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC9G,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AA1SD,gCA0SC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { SparseVector, SparseVectorConfig } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* Abstract interface for sparse vector generation
|
|
4
|
+
*
|
|
5
|
+
* Sparse vectors are used for:
|
|
6
|
+
* - BM25 keyword-based search
|
|
7
|
+
* - Hybrid search (combining dense + sparse)
|
|
8
|
+
* - Text-based retrieval with term importance
|
|
9
|
+
*
|
|
10
|
+
* Implementations should:
|
|
11
|
+
* 1. Learn vocabulary from a corpus
|
|
12
|
+
* 2. Generate sparse vectors with indices (vocabulary positions) and values (scores)
|
|
13
|
+
* 3. Support both single document and batch processing
|
|
14
|
+
*/
|
|
15
|
+
export interface SparseVectorGenerator {
|
|
16
|
+
/**
|
|
17
|
+
* Learn vocabulary and statistics from a corpus of documents
|
|
18
|
+
* This should be called during indexing to build IDF statistics
|
|
19
|
+
*
|
|
20
|
+
* @param documents - Array of document texts to learn from
|
|
21
|
+
*/
|
|
22
|
+
learn: (documents: string[]) => void;
|
|
23
|
+
/**
|
|
24
|
+
* Generate sparse vector for a single text
|
|
25
|
+
*
|
|
26
|
+
* @param text - Input text to vectorize
|
|
27
|
+
* @param config - Optional configuration for vector generation
|
|
28
|
+
* @returns Sparse vector with indices and values
|
|
29
|
+
*/
|
|
30
|
+
generate: (text: string, config?: SparseVectorConfig) => SparseVector;
|
|
31
|
+
/**
|
|
32
|
+
* Generate sparse vectors for multiple texts (batch operation)
|
|
33
|
+
*
|
|
34
|
+
* @param texts - Array of input texts to vectorize
|
|
35
|
+
* @param config - Optional configuration for vector generation
|
|
36
|
+
* @returns Array of sparse vectors
|
|
37
|
+
*/
|
|
38
|
+
generateBatch: (texts: string[], config?: SparseVectorConfig) => SparseVector[];
|
|
39
|
+
/**
|
|
40
|
+
* Get the vocabulary size (number of unique terms)
|
|
41
|
+
* Useful for debugging and understanding the sparse vector space
|
|
42
|
+
*/
|
|
43
|
+
getVocabularySize: () => number;
|
|
44
|
+
/**
|
|
45
|
+
* Get average document length from the learned corpus
|
|
46
|
+
* Used for BM25 length normalization
|
|
47
|
+
*/
|
|
48
|
+
getAverageDocumentLength: () => number;
|
|
49
|
+
/**
|
|
50
|
+
* Check if the generator has been trained (learned from corpus)
|
|
51
|
+
*/
|
|
52
|
+
isTrained: () => boolean;
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=sparse-vector-generator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sparse-vector-generator.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/sparse-vector-generator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEhE;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,qBAAqB;IACpC;;;;;OAKG;IACH,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;IAErC;;;;;;OAMG;IACH,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,kBAAkB,KAAK,YAAY,CAAC;IAEtE;;;;;;OAMG;IACH,aAAa,EAAE,CACb,KAAK,EAAE,MAAM,EAAE,EACf,MAAM,CAAC,EAAE,kBAAkB,KACxB,YAAY,EAAE,CAAC;IAEpB;;;OAGG;IACH,iBAAiB,EAAE,MAAM,MAAM,CAAC;IAEhC;;;OAGG;IACH,wBAAwB,EAAE,MAAM,MAAM,CAAC;IAEvC;;OAEG;IACH,SAAS,EAAE,MAAM,OAAO,CAAC;CAC1B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sparse-vector-generator.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/sparse-vector-generator.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sparse vector representation
|
|
3
|
+
* Used for BM25 and other sparse encoding methods
|
|
4
|
+
*/
|
|
5
|
+
export interface SparseVector {
|
|
6
|
+
/**
|
|
7
|
+
* Indices of non-zero elements
|
|
8
|
+
* Each index maps to a position in the vocabulary
|
|
9
|
+
*/
|
|
10
|
+
indices: number[];
|
|
11
|
+
/**
|
|
12
|
+
* Values corresponding to the indices
|
|
13
|
+
* Typically BM25 scores or TF-IDF weights
|
|
14
|
+
*/
|
|
15
|
+
values: number[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Configuration for sparse vector generation
|
|
19
|
+
*/
|
|
20
|
+
export interface SparseVectorConfig {
|
|
21
|
+
/**
|
|
22
|
+
* Minimum score threshold for including a term
|
|
23
|
+
* Terms with scores below this will be filtered out
|
|
24
|
+
* @default 0
|
|
25
|
+
*/
|
|
26
|
+
minScore?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Maximum number of terms to include in sparse vector
|
|
29
|
+
* @default undefined (no limit)
|
|
30
|
+
*/
|
|
31
|
+
maxTerms?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Whether to normalize vector values
|
|
34
|
+
* @default false
|
|
35
|
+
*/
|
|
36
|
+
normalize?: boolean;
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/vectordb/sparse/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,OAAO,EAAE,MAAM,EAAE,CAAC;IAElB;;;OAGG;IACH,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/vectordb/sparse/types.ts"],"names":[],"mappings":""}
|