langchain 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents.mjs +1 -0
- package/chains.mjs +1 -0
- package/dist/agents/tests/{agent.test.d.ts → agent.int.test.d.ts} +0 -0
- package/dist/agents/tests/{agent.test.js → agent.int.test.js} +2 -3
- package/dist/agents/tests/agent.int.test.js.map +1 -0
- package/dist/chains/base.d.ts +1 -0
- package/dist/chains/base.js +3 -2
- package/dist/chains/base.js.map +1 -1
- package/dist/chains/index.d.ts +1 -1
- package/dist/chains/index.js.map +1 -1
- package/dist/chains/llm_chain.d.ts +3 -2
- package/dist/chains/llm_chain.js +1 -1
- package/dist/chains/llm_chain.js.map +1 -1
- package/dist/chains/question_answering/tests/{load.test.d.ts → load.int.test.d.ts} +0 -0
- package/dist/chains/question_answering/tests/{load.test.js → load.int.test.js} +1 -1
- package/dist/chains/question_answering/tests/load.int.test.js.map +1 -0
- package/dist/chains/tests/{chat_vector_db_qa_chain.test.d.ts → chat_vector_db_qa_chain.int.test.d.ts} +0 -0
- package/dist/chains/tests/{chat_vector_db_qa_chain.test.js → chat_vector_db_qa_chain.int.test.js} +1 -1
- package/dist/chains/tests/chat_vector_db_qa_chain.int.test.js.map +1 -0
- package/dist/chains/tests/{combine_docs_chain.test.d.ts → combine_docs_chain.int.test.d.ts} +0 -0
- package/dist/chains/tests/{combine_docs_chain.test.js → combine_docs_chain.int.test.js} +1 -1
- package/dist/chains/tests/combine_docs_chain.int.test.js.map +1 -0
- package/dist/chains/tests/{llm_chain.test.d.ts → llm_chain.int.test.d.ts} +0 -0
- package/dist/chains/tests/{llm_chain.test.js → llm_chain.int.test.js} +1 -1
- package/dist/chains/tests/llm_chain.int.test.js.map +1 -0
- package/dist/chains/tests/{vector_db_qa_chain.test.d.ts → vector_db_qa_chain.int.test.d.ts} +0 -0
- package/dist/chains/tests/{vector_db_qa_chain.test.js → vector_db_qa_chain.int.test.js} +1 -1
- package/dist/chains/tests/vector_db_qa_chain.int.test.js.map +1 -0
- package/dist/docstore/base.d.ts +5 -0
- package/dist/docstore/base.js +7 -0
- package/dist/docstore/base.js.map +1 -0
- package/dist/docstore/in_memory.d.ts +10 -0
- package/dist/docstore/in_memory.js +36 -0
- package/dist/docstore/in_memory.js.map +1 -0
- package/dist/docstore/index.d.ts +3 -0
- package/dist/docstore/index.js +10 -0
- package/dist/docstore/index.js.map +1 -0
- package/dist/embeddings/fake.d.ts +5 -0
- package/dist/embeddings/fake.js +14 -0
- package/dist/embeddings/fake.js.map +1 -0
- package/dist/embeddings/tests/{openai.test.d.ts → openai.int.test.d.ts} +0 -0
- package/dist/embeddings/tests/{openai.test.js → openai.int.test.js} +1 -1
- package/dist/embeddings/tests/openai.int.test.js.map +1 -0
- package/dist/llms/base.js +4 -3
- package/dist/llms/base.js.map +1 -1
- package/dist/llms/index.d.ts +5 -4
- package/dist/llms/index.js +2 -1
- package/dist/llms/index.js.map +1 -1
- package/dist/llms/openai.d.ts +18 -2
- package/dist/llms/openai.js +111 -5
- package/dist/llms/openai.js.map +1 -1
- package/dist/llms/tests/{cohere.test.d.ts → cohere.int.test.d.ts} +0 -0
- package/dist/llms/tests/{cohere.test.js → cohere.int.test.js} +1 -1
- package/dist/llms/tests/cohere.int.test.js.map +1 -0
- package/dist/llms/tests/{openai.test.d.ts → openai.int.test.d.ts} +0 -0
- package/dist/llms/tests/{openai.test.js → openai.int.test.js} +1 -1
- package/dist/llms/tests/openai.int.test.js.map +1 -0
- package/dist/tests/text_splitter.test.d.ts +1 -0
- package/dist/tests/text_splitter.test.js +141 -0
- package/dist/tests/text_splitter.test.js.map +1 -0
- package/dist/text_splitter.d.ts +17 -0
- package/dist/text_splitter.js +69 -1
- package/dist/text_splitter.js.map +1 -1
- package/dist/util/hub.js +3 -2
- package/dist/util/hub.js.map +1 -1
- package/dist/vectorstores/base.d.ts +2 -6
- package/dist/vectorstores/base.js +2 -11
- package/dist/vectorstores/base.js.map +1 -1
- package/dist/vectorstores/hnswlib.d.ts +7 -4
- package/dist/vectorstores/hnswlib.js +24 -9
- package/dist/vectorstores/hnswlib.js.map +1 -1
- package/dist/vectorstores/index.d.ts +1 -0
- package/dist/vectorstores/index.js +3 -1
- package/dist/vectorstores/index.js.map +1 -1
- package/dist/vectorstores/pinecone.d.ts +17 -0
- package/dist/vectorstores/pinecone.js +78 -0
- package/dist/vectorstores/pinecone.js.map +1 -0
- package/dist/vectorstores/tests/hnswlib.int.test.d.ts +1 -0
- package/dist/vectorstores/tests/hnswlib.int.test.js +61 -0
- package/dist/vectorstores/tests/hnswlib.int.test.js.map +1 -0
- package/dist/vectorstores/tests/hnswlib.test.js +29 -38
- package/dist/vectorstores/tests/hnswlib.test.js.map +1 -1
- package/docstore.d.ts +1 -0
- package/docstore.js +1 -0
- package/docstore.mjs +1 -0
- package/document.mjs +1 -0
- package/embeddings.mjs +1 -0
- package/index.d.ts +1 -0
- package/index.js +1 -0
- package/index.mjs +1 -0
- package/llms.mjs +1 -0
- package/memory.mjs +1 -0
- package/package.json +111 -8
- package/prompts.mjs +1 -0
- package/text_splitter.mjs +1 -0
- package/tools.mjs +1 -0
- package/vectorstores.mjs +1 -0
- package/dist/agents/tests/agent.test.js.map +0 -1
- package/dist/chains/question_answering/tests/load.test.js.map +0 -1
- package/dist/chains/tests/chat_vector_db_qa_chain.test.js.map +0 -1
- package/dist/chains/tests/combine_docs_chain.test.js.map +0 -1
- package/dist/chains/tests/llm_chain.test.js.map +0 -1
- package/dist/chains/tests/vector_db_qa_chain.test.js.map +0 -1
- package/dist/embeddings/tests/openai.test.js.map +0 -1
- package/dist/llms/tests/cohere.test.js.map +0 -1
- package/dist/llms/tests/openai.test.js.map +0 -1
|
@@ -8,6 +8,7 @@ const promises_1 = __importDefault(require("fs/promises"));
|
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
9
|
const base_1 = require("./base");
|
|
10
10
|
const document_1 = require("../document");
|
|
11
|
+
const docstore_1 = require("../docstore");
|
|
11
12
|
let HierarchicalNSW = null;
|
|
12
13
|
try {
|
|
13
14
|
// eslint-disable-next-line global-require,import/no-extraneous-dependencies
|
|
@@ -18,13 +19,19 @@ catch (_a) {
|
|
|
18
19
|
}
|
|
19
20
|
class HNSWLib extends base_1.SaveableVectorStore {
|
|
20
21
|
constructor(args, embeddings, docstore, index) {
|
|
21
|
-
super();
|
|
22
|
+
super(embeddings);
|
|
22
23
|
Object.defineProperty(this, "index", {
|
|
23
24
|
enumerable: true,
|
|
24
25
|
configurable: true,
|
|
25
26
|
writable: true,
|
|
26
27
|
value: void 0
|
|
27
28
|
});
|
|
29
|
+
Object.defineProperty(this, "docstore", {
|
|
30
|
+
enumerable: true,
|
|
31
|
+
configurable: true,
|
|
32
|
+
writable: true,
|
|
33
|
+
value: void 0
|
|
34
|
+
});
|
|
28
35
|
Object.defineProperty(this, "args", {
|
|
29
36
|
enumerable: true,
|
|
30
37
|
configurable: true,
|
|
@@ -36,6 +43,10 @@ class HNSWLib extends base_1.SaveableVectorStore {
|
|
|
36
43
|
this.embeddings = embeddings;
|
|
37
44
|
this.docstore = docstore;
|
|
38
45
|
}
|
|
46
|
+
async addDocuments(documents) {
|
|
47
|
+
const texts = documents.map(({ pageContent }) => pageContent);
|
|
48
|
+
return this.addVectors(await this.embeddings.embedDocuments(texts), documents);
|
|
49
|
+
}
|
|
39
50
|
async addVectors(vectors, documents) {
|
|
40
51
|
if (vectors.length === 0) {
|
|
41
52
|
return;
|
|
@@ -63,11 +74,12 @@ class HNSWLib extends base_1.SaveableVectorStore {
|
|
|
63
74
|
const capacity = this.index.getMaxElements();
|
|
64
75
|
const needed = this.index.getCurrentCount() + vectors.length;
|
|
65
76
|
if (needed > capacity) {
|
|
66
|
-
this.index.resizeIndex(needed
|
|
77
|
+
this.index.resizeIndex(needed);
|
|
67
78
|
}
|
|
79
|
+
const docstoreSize = this.docstore.count;
|
|
68
80
|
for (let i = 0; i < vectors.length; i += 1) {
|
|
69
|
-
this.index.addPoint(vectors[i], i);
|
|
70
|
-
this.docstore[i]
|
|
81
|
+
this.index.addPoint(vectors[i], docstoreSize + i);
|
|
82
|
+
this.docstore.add({ [docstoreSize + i]: documents[i] });
|
|
71
83
|
}
|
|
72
84
|
}
|
|
73
85
|
async similaritySearchVectorWithScore(query, k) {
|
|
@@ -75,7 +87,10 @@ class HNSWLib extends base_1.SaveableVectorStore {
|
|
|
75
87
|
throw new Error("Vector store not initialised yet. Try calling `addTexts` first.");
|
|
76
88
|
}
|
|
77
89
|
const result = this.index.searchKnn(query, k);
|
|
78
|
-
return result.neighbors.map((docIndex, resultIndex) => [
|
|
90
|
+
return result.neighbors.map((docIndex, resultIndex) => [
|
|
91
|
+
this.docstore.search(String(docIndex)),
|
|
92
|
+
result.distances[resultIndex],
|
|
93
|
+
]);
|
|
79
94
|
}
|
|
80
95
|
async save(directory) {
|
|
81
96
|
if (!this.index) {
|
|
@@ -102,7 +117,7 @@ class HNSWLib extends base_1.SaveableVectorStore {
|
|
|
102
117
|
]);
|
|
103
118
|
return new HNSWLib(args, embeddings, docstore, index);
|
|
104
119
|
}
|
|
105
|
-
static async fromTexts(texts, metadatas, embeddings) {
|
|
120
|
+
static async fromTexts(texts, metadatas, embeddings, docstore = new docstore_1.InMemoryDocstore()) {
|
|
106
121
|
const docs = [];
|
|
107
122
|
for (let i = 0; i < texts.length; i += 1) {
|
|
108
123
|
const newDoc = new document_1.Document({
|
|
@@ -111,16 +126,16 @@ class HNSWLib extends base_1.SaveableVectorStore {
|
|
|
111
126
|
});
|
|
112
127
|
docs.push(newDoc);
|
|
113
128
|
}
|
|
114
|
-
return HNSWLib.fromDocuments(docs, embeddings);
|
|
129
|
+
return HNSWLib.fromDocuments(docs, embeddings, docstore);
|
|
115
130
|
}
|
|
116
|
-
static async fromDocuments(docs, embeddings) {
|
|
131
|
+
static async fromDocuments(docs, embeddings, docstore = new docstore_1.InMemoryDocstore()) {
|
|
117
132
|
if (HierarchicalNSW === null) {
|
|
118
133
|
throw new Error("Please install hnswlib-node as a dependency with, e.g. `npm install -S hnswlib-node`");
|
|
119
134
|
}
|
|
120
135
|
const args = {
|
|
121
136
|
space: "ip", // dot product
|
|
122
137
|
};
|
|
123
|
-
const instance = new this(args, embeddings,
|
|
138
|
+
const instance = new this(args, embeddings, docstore);
|
|
124
139
|
await instance.addDocuments(docs);
|
|
125
140
|
return instance;
|
|
126
141
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnswlib.js","sourceRoot":"","sources":["../../vectorstores/hnswlib.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA6B;AAC7B,gDAAwB;
|
|
1
|
+
{"version":3,"file":"hnswlib.js","sourceRoot":"","sources":["../../vectorstores/hnswlib.ts"],"names":[],"mappings":";;;;;;AAAA,2DAA6B;AAC7B,gDAAwB;AAMxB,iCAA6C;AAC7C,0CAAuC;AACvC,0CAA+C;AAE/C,IAAI,eAAe,GAAmC,IAAI,CAAC;AAE3D,IAAI;IACF,4EAA4E;IAC5E,CAAC,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC;CACjD;AAAC,WAAM;IACN,eAAe;CAChB;AAOD,MAAa,OAAQ,SAAQ,0BAAmB;IAO9C,YACE,IAAiB,EACjB,UAAsB,EACtB,QAA0B,EAC1B,KAAwB;QAExB,KAAK,CAAC,UAAU,CAAC,CAAC;QAZpB;;;;;WAAyB;QAEzB;;;;;WAA2B;QAE3B;;;;;WAAkB;QAShB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,UAAU,CACpB,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAC3C,SAAS,CACV,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAmB,EAAE,SAAqB;QACzD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,IAAI,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,SAAS,EAAE;gBACzC,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;aAC7C;YACD,IAAI,eAAe,KAAK,IAAI,EAAE;gBAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;aACH;YACD,IAAI,CAAC,KAAK,GAAG,IAAI,eAAe,CAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,aAAa,CACxB,CAAC;YACF,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;SACtC;QACD,qEAAqE;QACrE,oEAAoE;QACpE,uEAAuE;QACvE,+DAA+D;QAC/D,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;SACpE;QACD,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACjD,MAAM,IAAI,KAAK,CACb,kEAAkE,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,CAC7F,CAAC;SACH;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7D,IAAI,MAAM,GAAG,QAAQ,EAAE;YACrB,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;SAChC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YAC1C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,YAAY,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;SACzD;IACH,CAAC;IAED,KAAK,CAAC,+BAA+B,CAAC,KAAe,EAAE,CAAS;QAC9D,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9C,OAAO,MAAM,CAAC,SAAS,CAAC,GAAG,CACzB,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,CACxB;YACE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,CAAC,SAAS,CAAC,WAAW,CAAC;SACR,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YACf,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;SACH;QACD,MAAM,kBAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;YAC5D,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAC1B;YACD,MAAM,kBAAE,CAAC,SAAS,CAChB,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAC9B;SACF,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,UAAsB;QACzD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,MAAM,kBAAE,CAAC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,CAC7D,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACnC,kBAAE;iBACC,QAAQ,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,MAAM,CAAC;iBACvD,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;YACnB,KAAK,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACvD,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CACpB,KAAe,EACf,SAAmB,EACnB,UAAsB,EACtB,WAA6B,IAAI,2BAAgB,EAAE;QAEnD,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,mBAAQ,CAAC;gBAC1B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;gBACrB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnB;QACD,OAAO,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,IAAgB,EAChB,UAAsB,EACtB,WAA6B,IAAI,2BAAgB,EAAE;QAEnD,IAAI,eAAe,KAAK,IAAI,EAAE;YAC5B,MAAM,IAAI,KAAK,CACb,sFAAsF,CACvF,CAAC;SACH;QACD,MAAM,IAAI,GAAgB;YACxB,KAAK,EAAE,IAAI,EAAE,cAAc;SAC5B,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAjKD,0BAiKC"}
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.HNSWLib = void 0;
|
|
3
|
+
exports.PineconeStore = exports.HNSWLib = void 0;
|
|
4
4
|
var hnswlib_1 = require("./hnswlib");
|
|
5
5
|
Object.defineProperty(exports, "HNSWLib", { enumerable: true, get: function () { return hnswlib_1.HNSWLib; } });
|
|
6
|
+
var pinecone_1 = require("./pinecone");
|
|
7
|
+
Object.defineProperty(exports, "PineconeStore", { enumerable: true, get: function () { return pinecone_1.PineconeStore; } });
|
|
6
8
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../vectorstores/index.ts"],"names":[],"mappings":";;;AAAA,qCAAoC;AAA3B,kGAAA,OAAO,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../vectorstores/index.ts"],"names":[],"mappings":";;;AAAA,qCAAoC;AAA3B,kGAAA,OAAO,OAAA;AAChB,uCAA2C;AAAlC,yGAAA,aAAa,OAAA"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { PineconeClient } from "pinecone-client";
|
|
2
|
+
import { VectorStore } from "./base";
|
|
3
|
+
import { Embeddings } from "../embeddings/base";
|
|
4
|
+
import { Document } from "../document";
|
|
5
|
+
type PineconeMetadata = Record<string, any>;
|
|
6
|
+
export declare class PineconeStore extends VectorStore {
|
|
7
|
+
textKey: string;
|
|
8
|
+
pineconeClient: PineconeClient<PineconeMetadata>;
|
|
9
|
+
constructor(pineconeClient: PineconeClient<PineconeMetadata>, embeddings: Embeddings, textKey?: string);
|
|
10
|
+
addDocuments(documents: Document[], ids?: string[]): Promise<void>;
|
|
11
|
+
addVectors(vectors: number[][], documents: Document[], ids?: string[]): Promise<void>;
|
|
12
|
+
similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
|
|
13
|
+
static fromTexts(pineconeClient: PineconeClient<PineconeMetadata>, texts: string[], metadatas: object[], embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
|
|
14
|
+
static fromDocuments(pineconeClient: PineconeClient<PineconeMetadata>, docs: Document[], embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
|
|
15
|
+
static fromExistingIndex(pineconeClient: PineconeClient<PineconeMetadata>, embeddings: Embeddings, textKey?: string): Promise<PineconeStore>;
|
|
16
|
+
}
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PineconeStore = void 0;
|
|
4
|
+
const uuid_1 = require("uuid");
|
|
5
|
+
const base_1 = require("./base");
|
|
6
|
+
const document_1 = require("../document");
|
|
7
|
+
class PineconeStore extends base_1.VectorStore {
|
|
8
|
+
constructor(pineconeClient, embeddings, textKey = "text") {
|
|
9
|
+
super(embeddings);
|
|
10
|
+
Object.defineProperty(this, "textKey", {
|
|
11
|
+
enumerable: true,
|
|
12
|
+
configurable: true,
|
|
13
|
+
writable: true,
|
|
14
|
+
value: void 0
|
|
15
|
+
});
|
|
16
|
+
Object.defineProperty(this, "pineconeClient", {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
configurable: true,
|
|
19
|
+
writable: true,
|
|
20
|
+
value: void 0
|
|
21
|
+
});
|
|
22
|
+
this.pineconeClient = pineconeClient;
|
|
23
|
+
this.embeddings = embeddings;
|
|
24
|
+
this.textKey = textKey;
|
|
25
|
+
}
|
|
26
|
+
async addDocuments(documents, ids) {
|
|
27
|
+
const texts = documents.map(({ pageContent }) => pageContent);
|
|
28
|
+
return this.addVectors(await this.embeddings.embedDocuments(texts), documents, ids);
|
|
29
|
+
}
|
|
30
|
+
async addVectors(vectors, documents, ids) {
|
|
31
|
+
const documentIds = ids == null ? documents.map(() => (0, uuid_1.v4)()) : ids;
|
|
32
|
+
return this.pineconeClient.upsert({
|
|
33
|
+
vectors: vectors.map((values, idx) => ({
|
|
34
|
+
id: documentIds[idx],
|
|
35
|
+
metadata: {
|
|
36
|
+
...documents[idx].metadata,
|
|
37
|
+
[this.textKey]: documents[idx].pageContent,
|
|
38
|
+
},
|
|
39
|
+
values,
|
|
40
|
+
})),
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
async similaritySearchVectorWithScore(query, k) {
|
|
44
|
+
const results = await this.pineconeClient.query({
|
|
45
|
+
topK: k,
|
|
46
|
+
includeMetadata: true,
|
|
47
|
+
vector: query,
|
|
48
|
+
});
|
|
49
|
+
const result = [];
|
|
50
|
+
for (const res of results.matches) {
|
|
51
|
+
const { [this.textKey]: pageContent, ...metadata } = res.metadata;
|
|
52
|
+
result.push([new document_1.Document({ metadata, pageContent }), res.score]);
|
|
53
|
+
}
|
|
54
|
+
return result;
|
|
55
|
+
}
|
|
56
|
+
static async fromTexts(pineconeClient, texts, metadatas, embeddings, textKey = "text") {
|
|
57
|
+
const docs = [];
|
|
58
|
+
for (let i = 0; i < texts.length; i += 1) {
|
|
59
|
+
const newDoc = new document_1.Document({
|
|
60
|
+
pageContent: texts[i],
|
|
61
|
+
metadata: metadatas[i],
|
|
62
|
+
});
|
|
63
|
+
docs.push(newDoc);
|
|
64
|
+
}
|
|
65
|
+
return PineconeStore.fromDocuments(pineconeClient, docs, embeddings, textKey);
|
|
66
|
+
}
|
|
67
|
+
static async fromDocuments(pineconeClient, docs, embeddings, textKey = "text") {
|
|
68
|
+
const instance = new this(pineconeClient, embeddings, textKey);
|
|
69
|
+
await instance.addDocuments(docs);
|
|
70
|
+
return instance;
|
|
71
|
+
}
|
|
72
|
+
static async fromExistingIndex(pineconeClient, embeddings, textKey = "text") {
|
|
73
|
+
const instance = new this(pineconeClient, embeddings, textKey);
|
|
74
|
+
return instance;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
exports.PineconeStore = PineconeStore;
|
|
78
|
+
//# sourceMappingURL=pinecone.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pinecone.js","sourceRoot":"","sources":["../../vectorstores/pinecone.ts"],"names":[],"mappings":";;;AACA,+BAAoC;AAEpC,iCAAqC;AAErC,0CAAuC;AAKvC,MAAa,aAAc,SAAQ,kBAAW;IAK5C,YACE,cAAgD,EAChD,UAAsB,EACtB,OAAO,GAAG,MAAM;QAEhB,KAAK,CAAC,UAAU,CAAC,CAAC;QATpB;;;;;WAAgB;QAEhB;;;;;WAAiD;QAS/C,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAqB,EAAE,GAAc;QACtD,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC;QAC9D,OAAO,IAAI,CAAC,UAAU,CACpB,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAC3C,SAAS,EACT,GAAG,CACJ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAAmB,EAAE,SAAqB,EAAE,GAAc;QACzE,MAAM,WAAW,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAA,SAAM,GAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QAEtE,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;YAChC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBACrC,EAAE,EAAE,WAAW,CAAC,GAAG,CAAC;gBACpB,QAAQ,EAAE;oBACR,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ;oBAC1B,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,WAAW;iBAC3C;gBACD,MAAM;aACP,CAAC,CAAC;SACJ,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,+BAA+B,CACnC,KAAe,EACf,CAAS;QAET,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC;YAC9C,IAAI,EAAE,CAAC;YACP,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QAEH,MAAM,MAAM,GAAyB,EAAE,CAAC;QAExC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,OAAO,EAAE;YACjC,MAAM,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,GAAG,QAAQ,EAAE,GAChD,GAAG,CAAC,QAA4B,CAAC;YACnC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,mBAAQ,CAAC,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;SACnE;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CACpB,cAAgD,EAChD,KAAe,EACf,SAAmB,EACnB,UAAsB,EACtB,OAAO,GAAG,MAAM;QAEhB,MAAM,IAAI,GAAG,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,mBAAQ,CAAC;gBAC1B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;gBACrB,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;aACvB,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;SACnB;QAED,OAAO,aAAa,CAAC,aAAa,CAChC,cAAc,EACd,IAAI,EACJ,UAAU,EACV,OAAO,CACR,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,aAAa,CACxB,cAAgD,EAChD,IAAgB,EAChB,UAAsB,EACtB,OAAO,GAAG,MAAM;QAEhB,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAC5B,cAAgD,EAChD,UAAsB,EACtB,OAAO,GAAG,MAAM;QAEhB,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF;AAzGD,sCAyGC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const globals_1 = require("@jest/globals");
|
|
7
|
+
const promises_1 = __importDefault(require("fs/promises"));
|
|
8
|
+
const path_1 = __importDefault(require("path"));
|
|
9
|
+
const os_1 = __importDefault(require("os"));
|
|
10
|
+
const hnswlib_1 = require("../hnswlib");
|
|
11
|
+
const embeddings_1 = require("../../embeddings");
|
|
12
|
+
const document_1 = require("../../document");
|
|
13
|
+
(0, globals_1.test)("Test HNSWLib.fromTexts", async () => {
|
|
14
|
+
var _a;
|
|
15
|
+
const vectorStore = await hnswlib_1.HNSWLib.fromTexts(["Hello world", "Bye bye", "hello nice world"], [{ id: 2 }, { id: 1 }, { id: 3 }], new embeddings_1.OpenAIEmbeddings());
|
|
16
|
+
(0, globals_1.expect)((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getCurrentCount()).toBe(3);
|
|
17
|
+
const resultOne = await vectorStore.similaritySearch("hello world", 1);
|
|
18
|
+
const resultOneMetadatas = resultOne.map(({ metadata }) => metadata);
|
|
19
|
+
(0, globals_1.expect)(resultOneMetadatas).toEqual([{ id: 2 }]);
|
|
20
|
+
const resultTwo = await vectorStore.similaritySearch("hello world", 3);
|
|
21
|
+
const resultTwoMetadatas = resultTwo.map(({ metadata }) => metadata);
|
|
22
|
+
(0, globals_1.expect)(resultTwoMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
|
|
23
|
+
});
|
|
24
|
+
(0, globals_1.test)("Test HNSWLib.fromTexts + addDocuments", async () => {
|
|
25
|
+
var _a, _b, _c;
|
|
26
|
+
const vectorStore = await hnswlib_1.HNSWLib.fromTexts(["Hello world", "Bye bye", "hello nice world"], [{ id: 2 }, { id: 1 }, { id: 3 }], new embeddings_1.OpenAIEmbeddings());
|
|
27
|
+
(0, globals_1.expect)((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getMaxElements()).toBe(3);
|
|
28
|
+
(0, globals_1.expect)((_b = vectorStore.index) === null || _b === void 0 ? void 0 : _b.getCurrentCount()).toBe(3);
|
|
29
|
+
await vectorStore.addDocuments([
|
|
30
|
+
new document_1.Document({
|
|
31
|
+
pageContent: "hello worldklmslksmn",
|
|
32
|
+
metadata: { id: 4 },
|
|
33
|
+
}),
|
|
34
|
+
]);
|
|
35
|
+
(0, globals_1.expect)((_c = vectorStore.index) === null || _c === void 0 ? void 0 : _c.getMaxElements()).toBe(4);
|
|
36
|
+
const resultTwo = await vectorStore.similaritySearch("hello world", 3);
|
|
37
|
+
const resultTwoMetadatas = resultTwo.map(({ metadata }) => metadata);
|
|
38
|
+
(0, globals_1.expect)(resultTwoMetadatas).toEqual([{ id: 2 }, { id: 1 }, { id: 3 }]);
|
|
39
|
+
});
|
|
40
|
+
(0, globals_1.test)("Test HNSWLib.load and HNSWLib.save", async () => {
|
|
41
|
+
var _a;
|
|
42
|
+
const vectorStore = await hnswlib_1.HNSWLib.fromTexts(["Hello world", "Bye bye", "hello nice world"], [{ id: 2 }, { id: 1 }, { id: 3 }], new embeddings_1.OpenAIEmbeddings());
|
|
43
|
+
(0, globals_1.expect)((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getCurrentCount()).toBe(3);
|
|
44
|
+
const resultOne = await vectorStore.similaritySearch("hello world", 1);
|
|
45
|
+
const resultOneMetadatas = resultOne.map(({ metadata }) => metadata);
|
|
46
|
+
(0, globals_1.expect)(resultOneMetadatas).toEqual([{ id: 2 }]);
|
|
47
|
+
const resultTwo = await vectorStore.similaritySearch("hello world", 3);
|
|
48
|
+
const resultTwoMetadatas = resultTwo.map(({ metadata }) => metadata);
|
|
49
|
+
(0, globals_1.expect)(resultTwoMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
|
|
50
|
+
const tempDirectory = await promises_1.default.mkdtemp(path_1.default.join(os_1.default.tmpdir(), "lcjs-"));
|
|
51
|
+
console.log(tempDirectory);
|
|
52
|
+
await vectorStore.save(tempDirectory);
|
|
53
|
+
const loadedVectorStore = await hnswlib_1.HNSWLib.load(tempDirectory, new embeddings_1.OpenAIEmbeddings());
|
|
54
|
+
const resultThree = await loadedVectorStore.similaritySearch("hello world", 1);
|
|
55
|
+
const resultThreeMetadatas = resultThree.map(({ metadata }) => metadata);
|
|
56
|
+
(0, globals_1.expect)(resultThreeMetadatas).toEqual([{ id: 2 }]);
|
|
57
|
+
const resultFour = await loadedVectorStore.similaritySearch("hello world", 3);
|
|
58
|
+
const resultFourMetadatas = resultFour.map(({ metadata }) => metadata);
|
|
59
|
+
(0, globals_1.expect)(resultFourMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
|
|
60
|
+
});
|
|
61
|
+
//# sourceMappingURL=hnswlib.int.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnswlib.int.test.js","sourceRoot":"","sources":["../../../vectorstores/tests/hnswlib.int.test.ts"],"names":[],"mappings":";;;;;AAAA,2CAA6C;AAC7C,2DAA6B;AAC7B,gDAAwB;AACxB,4CAAoB;AAEpB,wCAAqC;AACrC,iDAAoD;AACpD,6CAA0C;AAE1C,IAAA,cAAI,EAAC,wBAAwB,EAAE,KAAK,IAAI,EAAE;;IACxC,MAAM,WAAW,GAAG,MAAM,iBAAO,CAAC,SAAS,CACzC,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,CAAC,EAC9C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACjC,IAAI,6BAAgB,EAAE,CACvB,CAAC;IACF,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;;IACvD,MAAM,WAAW,GAAG,MAAM,iBAAO,CAAC,SAAS,CACzC,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,CAAC,EAC9C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACjC,IAAI,6BAAgB,EAAE,CACvB,CAAC;IACF,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,WAAW,CAAC,YAAY,CAAC;QAC7B,IAAI,mBAAQ,CAAC;YACX,WAAW,EAAE,sBAAsB;YACnC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;SACpB,CAAC;KACH,CAAC,CAAC;IACH,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC,CAAC;AAEH,IAAA,cAAI,EAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;;IACpD,MAAM,WAAW,GAAG,MAAM,iBAAO,CAAC,SAAS,CACzC,CAAC,aAAa,EAAE,SAAS,EAAE,kBAAkB,CAAC,EAC9C,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACjC,IAAI,6BAAgB,EAAE,CACvB,CAAC;IACF,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEhD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtE,MAAM,aAAa,GAAG,MAAM,kBAAE,CAAC,OAAO,CAAC,cAAI,CAAC,IAAI,CAAC,YAAE,CAAC,MAAM,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;IAExE,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAE3B,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAEtC,MAAM,iBAAiB,GAAG,MAAM,iBAAO,CAAC,IAAI,CAC1C,aAAa,EACb,IAAI,6BAAgB,EAAE,CACvB,CAAC;IAEF,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,CAC1D,aAAa,EACb,CAAC,CACF,CAAC;IAEF,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACzE,IAAA,gBAAM,EAAC,oBAAoB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAElD,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,gBAAgB,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;IAE9E,MAAM,mBAAmB,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACvE,IAAA,gBAAM,EAAC,mBAAmB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACzE,CAAC,CAAC,CAAC"}
|
|
@@ -1,44 +1,35 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
const globals_1 = require("@jest/globals");
|
|
7
|
-
const promises_1 = __importDefault(require("fs/promises"));
|
|
8
|
-
const path_1 = __importDefault(require("path"));
|
|
9
|
-
const os_1 = __importDefault(require("os"));
|
|
10
4
|
const hnswlib_1 = require("../hnswlib");
|
|
11
|
-
const
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
const
|
|
38
|
-
const
|
|
39
|
-
(0, globals_1.expect)(
|
|
40
|
-
const resultFour = await loadedVectorStore.similaritySearch("hello world", 3);
|
|
41
|
-
const resultFourMetadatas = resultFour.map(({ metadata }) => metadata);
|
|
42
|
-
(0, globals_1.expect)(resultFourMetadatas).toEqual([{ id: 2 }, { id: 3 }, { id: 1 }]);
|
|
5
|
+
const document_1 = require("../../document");
|
|
6
|
+
const fake_1 = require("../../embeddings/fake");
|
|
7
|
+
(0, globals_1.test)("Test HNSWLib.fromTexts + addVectors", async () => {
|
|
8
|
+
var _a, _b, _c;
|
|
9
|
+
const vectorStore = await hnswlib_1.HNSWLib.fromTexts(["Hello world"], [{ id: 2 }], new fake_1.FakeEmbeddings());
|
|
10
|
+
(0, globals_1.expect)((_a = vectorStore.index) === null || _a === void 0 ? void 0 : _a.getMaxElements()).toBe(1);
|
|
11
|
+
(0, globals_1.expect)((_b = vectorStore.index) === null || _b === void 0 ? void 0 : _b.getCurrentCount()).toBe(1);
|
|
12
|
+
await vectorStore.addVectors([
|
|
13
|
+
[0, 1, 0, 0],
|
|
14
|
+
[1, 0, 0, 0],
|
|
15
|
+
[0.5, 0.5, 0.5, 0.5],
|
|
16
|
+
], [
|
|
17
|
+
new document_1.Document({
|
|
18
|
+
pageContent: "hello bye",
|
|
19
|
+
metadata: { id: 5 },
|
|
20
|
+
}),
|
|
21
|
+
new document_1.Document({
|
|
22
|
+
pageContent: "hello worlddwkldnsk",
|
|
23
|
+
metadata: { id: 4 },
|
|
24
|
+
}),
|
|
25
|
+
new document_1.Document({
|
|
26
|
+
pageContent: "hello you",
|
|
27
|
+
metadata: { id: 6 },
|
|
28
|
+
}),
|
|
29
|
+
]);
|
|
30
|
+
(0, globals_1.expect)((_c = vectorStore.index) === null || _c === void 0 ? void 0 : _c.getMaxElements()).toBe(4);
|
|
31
|
+
const resultTwo = await vectorStore.similaritySearchVectorWithScore([1, 0, 0, 0], 3);
|
|
32
|
+
const resultTwoMetadatas = resultTwo.map(([{ metadata }]) => metadata);
|
|
33
|
+
(0, globals_1.expect)(resultTwoMetadatas).toEqual([{ id: 4 }, { id: 6 }, { id: 2 }]);
|
|
43
34
|
});
|
|
44
35
|
//# sourceMappingURL=hnswlib.test.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnswlib.test.js","sourceRoot":"","sources":["../../../vectorstores/tests/hnswlib.test.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hnswlib.test.js","sourceRoot":"","sources":["../../../vectorstores/tests/hnswlib.test.ts"],"names":[],"mappings":";;AAAA,2CAA6C;AAC7C,wCAAqC;AACrC,6CAA0C;AAC1C,gDAAuD;AAEvD,IAAA,cAAI,EAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;;IACrD,MAAM,WAAW,GAAG,MAAM,iBAAO,CAAC,SAAS,CACzC,CAAC,aAAa,CAAC,EACf,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,EACX,IAAI,qBAAc,EAAE,CACrB,CAAC;IACF,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpD,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAErD,MAAM,WAAW,CAAC,UAAU,CAC1B;QACE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACZ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACZ,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;KACrB,EACD;QACE,IAAI,mBAAQ,CAAC;YACX,WAAW,EAAE,WAAW;YACxB,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;SACpB,CAAC;QACF,IAAI,mBAAQ,CAAC;YACX,WAAW,EAAE,qBAAqB;YAClC,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;SACpB,CAAC;QACF,IAAI,mBAAQ,CAAC;YACX,WAAW,EAAE,WAAW;YACxB,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE;SACpB,CAAC;KACH,CACF,CAAC;IACF,IAAA,gBAAM,EAAC,MAAA,WAAW,CAAC,KAAK,0CAAE,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,+BAA+B,CACjE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EACZ,CAAC,CACF,CAAC;IACF,MAAM,kBAAkB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC;IACvE,IAAA,gBAAM,EAAC,kBAAkB,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC,CAAC"}
|
package/docstore.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/docstore'
|
package/docstore.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('./dist/docstore')
|
package/docstore.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/docstore/index.js'
|
package/document.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/document.js'
|
package/embeddings.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/embeddings/index.js'
|
package/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/.'
|
package/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
module.exports = require('./dist/.')
|
package/index.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/index.js'
|
package/llms.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/llms/index.js'
|
package/memory.mjs
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from './dist/memory/index.js'
|
package/package.json
CHANGED
|
@@ -1,31 +1,47 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "langchain",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"description": "Typescript bindings for langchain",
|
|
5
|
-
"main": "./
|
|
6
|
-
"types": "./
|
|
5
|
+
"main": "./index.js",
|
|
6
|
+
"types": "./index.d.ts",
|
|
7
7
|
"files": [
|
|
8
8
|
"dist/",
|
|
9
9
|
"agents.js",
|
|
10
|
+
"agents.mjs",
|
|
10
11
|
"agents.d.ts",
|
|
11
12
|
"tools.js",
|
|
13
|
+
"tools.mjs",
|
|
12
14
|
"tools.d.ts",
|
|
13
15
|
"chains.js",
|
|
16
|
+
"chains.mjs",
|
|
14
17
|
"chains.d.ts",
|
|
15
18
|
"embeddings.js",
|
|
19
|
+
"embeddings.mjs",
|
|
16
20
|
"embeddings.d.ts",
|
|
17
21
|
"llms.js",
|
|
22
|
+
"llms.mjs",
|
|
18
23
|
"llms.d.ts",
|
|
19
24
|
"prompts.js",
|
|
25
|
+
"prompts.mjs",
|
|
20
26
|
"prompts.d.ts",
|
|
21
27
|
"vectorstores.js",
|
|
28
|
+
"vectorstores.mjs",
|
|
22
29
|
"vectorstores.d.ts",
|
|
23
30
|
"text_splitter.js",
|
|
31
|
+
"text_splitter.mjs",
|
|
24
32
|
"text_splitter.d.ts",
|
|
25
33
|
"memory.js",
|
|
34
|
+
"memory.mjs",
|
|
26
35
|
"memory.d.ts",
|
|
27
36
|
"document.js",
|
|
28
|
-
"document.
|
|
37
|
+
"document.mjs",
|
|
38
|
+
"document.d.ts",
|
|
39
|
+
"docstore.js",
|
|
40
|
+
"docstore.mjs",
|
|
41
|
+
"docstore.d.ts",
|
|
42
|
+
"index.js",
|
|
43
|
+
"index.mjs",
|
|
44
|
+
"index.d.ts"
|
|
29
45
|
],
|
|
30
46
|
"repository": {
|
|
31
47
|
"type": "git",
|
|
@@ -39,17 +55,23 @@
|
|
|
39
55
|
"precommit": "tsc --noEmit && lint-staged",
|
|
40
56
|
"clean": "rm -rf dist/ && node create-entrypoints.js clean",
|
|
41
57
|
"prepack": "yarn build",
|
|
42
|
-
"test": "
|
|
43
|
-
"
|
|
58
|
+
"test": "yarn clean && jest --testPathIgnorePatterns=\\.int\\.test.ts",
|
|
59
|
+
"test:integration": "yarn clean && jest --testPathPattern=\\.int\\.test.ts",
|
|
60
|
+
"prepare": "husky install",
|
|
61
|
+
"format": "prettier --write \"**/*.ts\"",
|
|
62
|
+
"format:diff": "prettier --list-different \"**/*.ts\"",
|
|
63
|
+
"ci": "yarn lint && yarn format:diff && yarn build"
|
|
44
64
|
},
|
|
45
65
|
"author": "Langchain",
|
|
46
66
|
"license": "MIT",
|
|
47
67
|
"devDependencies": {
|
|
48
68
|
"@babel/core": "^7.20.12",
|
|
49
69
|
"@babel/preset-env": "^7.20.2",
|
|
70
|
+
"@dqbd/tiktoken": "^0.2.1",
|
|
50
71
|
"@jest/globals": "^29.4.2",
|
|
51
72
|
"@tsconfig/recommended": "^1.0.2",
|
|
52
73
|
"@types/node-fetch": "2",
|
|
74
|
+
"@types/uuid": "^9",
|
|
53
75
|
"@typescript-eslint/eslint-plugin": "^5.51.0",
|
|
54
76
|
"@typescript-eslint/parser": "^5.51.0",
|
|
55
77
|
"babel-jest": "^29.4.2",
|
|
@@ -60,11 +82,12 @@
|
|
|
60
82
|
"eslint-config-prettier": "^8.6.0",
|
|
61
83
|
"eslint-plugin-import": "^2.27.5",
|
|
62
84
|
"eslint-plugin-prettier": "^4.2.1",
|
|
63
|
-
"hnswlib-node": "^1.
|
|
85
|
+
"hnswlib-node": "^1.3.0",
|
|
64
86
|
"husky": "^8.0.3",
|
|
65
87
|
"jest": "^29.4.2",
|
|
66
88
|
"lint-staged": "^13.1.1",
|
|
67
89
|
"openai": "^3.1.0",
|
|
90
|
+
"pinecone-client": "^1.0.1",
|
|
68
91
|
"prettier": "^2.8.3",
|
|
69
92
|
"serpapi": "^1.1.1",
|
|
70
93
|
"ts-jest": "^29.0.5",
|
|
@@ -73,11 +96,41 @@
|
|
|
73
96
|
"typedoc-plugin-missing-exports": "^1.0.0",
|
|
74
97
|
"typescript": "^4.9.5"
|
|
75
98
|
},
|
|
99
|
+
"peerDependencies": {
|
|
100
|
+
"@dqbd/tiktoken": "^0.2.1",
|
|
101
|
+
"cohere-ai": "^5.0.2",
|
|
102
|
+
"hnswlib-node": "^1.3.0",
|
|
103
|
+
"openai": "^3.1.0",
|
|
104
|
+
"pinecone-client": "^1.0.1",
|
|
105
|
+
"serpapi": "^1.1.1"
|
|
106
|
+
},
|
|
107
|
+
"peerDependenciesMeta": {
|
|
108
|
+
"@dqbd/tiktoken": {
|
|
109
|
+
"optional": true
|
|
110
|
+
},
|
|
111
|
+
"cohere-ai": {
|
|
112
|
+
"optional": true
|
|
113
|
+
},
|
|
114
|
+
"hnswlib-node": {
|
|
115
|
+
"optional": true
|
|
116
|
+
},
|
|
117
|
+
"openai": {
|
|
118
|
+
"optional": true
|
|
119
|
+
},
|
|
120
|
+
"pinecone-client": {
|
|
121
|
+
"optional": true
|
|
122
|
+
},
|
|
123
|
+
"serpapi": {
|
|
124
|
+
"optional": true
|
|
125
|
+
}
|
|
126
|
+
},
|
|
76
127
|
"dependencies": {
|
|
77
128
|
"deepcopy": "^2.1.0",
|
|
129
|
+
"eventsource-parser": "^0.1.0",
|
|
78
130
|
"exponential-backoff": "^3.1.0",
|
|
79
131
|
"expr-eval": "^2.0.2",
|
|
80
132
|
"node-fetch": "2",
|
|
133
|
+
"uuid": "^9.0.0",
|
|
81
134
|
"yaml": "^2.2.1"
|
|
82
135
|
},
|
|
83
136
|
"lint-staged": {
|
|
@@ -102,5 +155,55 @@
|
|
|
102
155
|
"openai",
|
|
103
156
|
"embeddings",
|
|
104
157
|
"vectorstores"
|
|
105
|
-
]
|
|
158
|
+
],
|
|
159
|
+
"exports": {
|
|
160
|
+
".": {
|
|
161
|
+
"import": "./index.mjs",
|
|
162
|
+
"default": "./index.js"
|
|
163
|
+
},
|
|
164
|
+
"./agents": {
|
|
165
|
+
"import": "./agents.mjs",
|
|
166
|
+
"default": "./agents.js"
|
|
167
|
+
},
|
|
168
|
+
"./tools": {
|
|
169
|
+
"import": "./tools.mjs",
|
|
170
|
+
"default": "./tools.js"
|
|
171
|
+
},
|
|
172
|
+
"./chains": {
|
|
173
|
+
"import": "./chains.mjs",
|
|
174
|
+
"default": "./chains.js"
|
|
175
|
+
},
|
|
176
|
+
"./embeddings": {
|
|
177
|
+
"import": "./embeddings.mjs",
|
|
178
|
+
"default": "./embeddings.js"
|
|
179
|
+
},
|
|
180
|
+
"./llms": {
|
|
181
|
+
"import": "./llms.mjs",
|
|
182
|
+
"default": "./llms.js"
|
|
183
|
+
},
|
|
184
|
+
"./prompts": {
|
|
185
|
+
"import": "./prompts.mjs",
|
|
186
|
+
"default": "./prompts.js"
|
|
187
|
+
},
|
|
188
|
+
"./vectorstores": {
|
|
189
|
+
"import": "./vectorstores.mjs",
|
|
190
|
+
"default": "./vectorstores.js"
|
|
191
|
+
},
|
|
192
|
+
"./text_splitter": {
|
|
193
|
+
"import": "./text_splitter.mjs",
|
|
194
|
+
"default": "./text_splitter.js"
|
|
195
|
+
},
|
|
196
|
+
"./memory": {
|
|
197
|
+
"import": "./memory.mjs",
|
|
198
|
+
"default": "./memory.js"
|
|
199
|
+
},
|
|
200
|
+
"./document": {
|
|
201
|
+
"import": "./document.mjs",
|
|
202
|
+
"default": "./document.js"
|
|
203
|
+
},
|
|
204
|
+
"./docstore": {
|
|
205
|
+
"import": "./docstore.mjs",
|
|
206
|
+
"default": "./docstore.js"
|
|
207
|
+
}
|
|
208
|
+
}
|
|
106
209
|
}
|