langchain 0.0.170 → 0.0.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/chat_models/llama_cpp.cjs +1 -0
- package/chat_models/llama_cpp.d.ts +1 -0
- package/chat_models/llama_cpp.js +1 -0
- package/dist/agents/toolkits/openapi/openapi.cjs +1 -1
- package/dist/agents/toolkits/openapi/openapi.d.ts +1 -1
- package/dist/agents/toolkits/openapi/openapi.js +1 -1
- package/dist/chains/sql_db/sql_db_chain.cjs +2 -0
- package/dist/chains/sql_db/sql_db_chain.d.ts +2 -0
- package/dist/chains/sql_db/sql_db_chain.js +2 -0
- package/dist/chat_models/llama_cpp.cjs +243 -0
- package/dist/chat_models/llama_cpp.d.ts +94 -0
- package/dist/chat_models/llama_cpp.js +239 -0
- package/dist/document_loaders/web/pdf.cjs +23 -5
- package/dist/document_loaders/web/pdf.d.ts +9 -1
- package/dist/document_loaders/web/pdf.js +20 -2
- package/dist/graphs/neo4j_graph.cjs +14 -0
- package/dist/graphs/neo4j_graph.d.ts +14 -0
- package/dist/graphs/neo4j_graph.js +14 -0
- package/dist/llms/googlepalm.cjs +3 -0
- package/dist/llms/googlepalm.js +3 -0
- package/dist/load/import_constants.cjs +1 -0
- package/dist/load/import_constants.js +1 -0
- package/dist/retrievers/parent_document.cjs +22 -2
- package/dist/retrievers/parent_document.d.ts +8 -1
- package/dist/retrievers/parent_document.js +22 -2
- package/dist/schema/runnable/passthrough.cjs +3 -1
- package/dist/schema/runnable/passthrough.js +3 -1
- package/dist/sql_db.cjs +2 -0
- package/dist/sql_db.d.ts +2 -0
- package/dist/sql_db.js +2 -0
- package/dist/util/stream.cjs +3 -0
- package/dist/util/stream.js +3 -0
- package/dist/vectorstores/cassandra.cjs +25 -4
- package/dist/vectorstores/cassandra.d.ts +11 -1
- package/dist/vectorstores/cassandra.js +25 -4
- package/dist/vectorstores/momento_vector_index.cjs +3 -15
- package/dist/vectorstores/momento_vector_index.d.ts +0 -8
- package/dist/vectorstores/momento_vector_index.js +3 -15
- package/dist/vectorstores/neo4j_vector.cjs +14 -0
- package/dist/vectorstores/neo4j_vector.d.ts +14 -0
- package/dist/vectorstores/neo4j_vector.js +14 -0
- package/package.json +15 -7
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WebPDFLoader = void 0;
|
|
4
|
-
const pdf_js_1 = require("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
|
|
5
4
|
const document_js_1 = require("../../document.cjs");
|
|
6
5
|
const base_js_1 = require("../base.cjs");
|
|
7
6
|
/**
|
|
8
7
|
* A document loader for loading data from PDFs.
|
|
9
8
|
*/
|
|
10
9
|
class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
11
|
-
constructor(blob, { splitPages = true } = {}) {
|
|
10
|
+
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports } = {}) {
|
|
12
11
|
super();
|
|
13
12
|
Object.defineProperty(this, "blob", {
|
|
14
13
|
enumerable: true,
|
|
@@ -22,15 +21,23 @@ class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
|
22
21
|
writable: true,
|
|
23
22
|
value: true
|
|
24
23
|
});
|
|
24
|
+
Object.defineProperty(this, "pdfjs", {
|
|
25
|
+
enumerable: true,
|
|
26
|
+
configurable: true,
|
|
27
|
+
writable: true,
|
|
28
|
+
value: void 0
|
|
29
|
+
});
|
|
25
30
|
this.blob = blob;
|
|
26
31
|
this.splitPages = splitPages ?? this.splitPages;
|
|
32
|
+
this.pdfjs = pdfjs;
|
|
27
33
|
}
|
|
28
34
|
/**
|
|
29
35
|
* Loads the contents of the PDF as documents.
|
|
30
36
|
* @returns An array of Documents representing the retrieved data.
|
|
31
37
|
*/
|
|
32
38
|
async load() {
|
|
33
|
-
const
|
|
39
|
+
const { getDocument, version } = await this.pdfjs();
|
|
40
|
+
const parsedPdf = await getDocument({
|
|
34
41
|
data: new Uint8Array(await this.blob.arrayBuffer()),
|
|
35
42
|
useWorkerFetch: false,
|
|
36
43
|
isEvalSupported: false,
|
|
@@ -51,7 +58,7 @@ class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
|
51
58
|
pageContent: text,
|
|
52
59
|
metadata: {
|
|
53
60
|
pdf: {
|
|
54
|
-
version
|
|
61
|
+
version,
|
|
55
62
|
info: meta?.info,
|
|
56
63
|
metadata: meta?.metadata,
|
|
57
64
|
totalPages: parsedPdf.numPages,
|
|
@@ -73,7 +80,7 @@ class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
|
73
80
|
pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
|
|
74
81
|
metadata: {
|
|
75
82
|
pdf: {
|
|
76
|
-
version
|
|
83
|
+
version,
|
|
77
84
|
info: meta?.info,
|
|
78
85
|
metadata: meta?.metadata,
|
|
79
86
|
totalPages: parsedPdf.numPages,
|
|
@@ -85,3 +92,14 @@ class WebPDFLoader extends base_js_1.BaseDocumentLoader {
|
|
|
85
92
|
}
|
|
86
93
|
}
|
|
87
94
|
exports.WebPDFLoader = WebPDFLoader;
|
|
95
|
+
async function PDFLoaderImports() {
|
|
96
|
+
try {
|
|
97
|
+
const { default: mod } = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
|
|
98
|
+
const { getDocument, version } = mod;
|
|
99
|
+
return { getDocument, version };
|
|
100
|
+
}
|
|
101
|
+
catch (e) {
|
|
102
|
+
console.error(e);
|
|
103
|
+
throw new Error("Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.");
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
/// <reference path="../../../src/types/pdf-parse.d.ts" />
|
|
1
2
|
import { Document } from "../../document.js";
|
|
2
3
|
import { BaseDocumentLoader } from "../base.js";
|
|
3
4
|
/**
|
|
@@ -6,8 +7,10 @@ import { BaseDocumentLoader } from "../base.js";
|
|
|
6
7
|
export declare class WebPDFLoader extends BaseDocumentLoader {
|
|
7
8
|
protected blob: Blob;
|
|
8
9
|
protected splitPages: boolean;
|
|
9
|
-
|
|
10
|
+
private pdfjs;
|
|
11
|
+
constructor(blob: Blob, { splitPages, pdfjs }?: {
|
|
10
12
|
splitPages?: boolean | undefined;
|
|
13
|
+
pdfjs?: typeof PDFLoaderImports | undefined;
|
|
11
14
|
});
|
|
12
15
|
/**
|
|
13
16
|
* Loads the contents of the PDF as documents.
|
|
@@ -15,3 +18,8 @@ export declare class WebPDFLoader extends BaseDocumentLoader {
|
|
|
15
18
|
*/
|
|
16
19
|
load(): Promise<Document[]>;
|
|
17
20
|
}
|
|
21
|
+
declare function PDFLoaderImports(): Promise<{
|
|
22
|
+
getDocument: typeof import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js").getDocument;
|
|
23
|
+
version: string;
|
|
24
|
+
}>;
|
|
25
|
+
export {};
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import { getDocument, version, } from "pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js";
|
|
2
1
|
import { Document } from "../../document.js";
|
|
3
2
|
import { BaseDocumentLoader } from "../base.js";
|
|
4
3
|
/**
|
|
5
4
|
* A document loader for loading data from PDFs.
|
|
6
5
|
*/
|
|
7
6
|
export class WebPDFLoader extends BaseDocumentLoader {
|
|
8
|
-
constructor(blob, { splitPages = true } = {}) {
|
|
7
|
+
constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports } = {}) {
|
|
9
8
|
super();
|
|
10
9
|
Object.defineProperty(this, "blob", {
|
|
11
10
|
enumerable: true,
|
|
@@ -19,14 +18,22 @@ export class WebPDFLoader extends BaseDocumentLoader {
|
|
|
19
18
|
writable: true,
|
|
20
19
|
value: true
|
|
21
20
|
});
|
|
21
|
+
Object.defineProperty(this, "pdfjs", {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
configurable: true,
|
|
24
|
+
writable: true,
|
|
25
|
+
value: void 0
|
|
26
|
+
});
|
|
22
27
|
this.blob = blob;
|
|
23
28
|
this.splitPages = splitPages ?? this.splitPages;
|
|
29
|
+
this.pdfjs = pdfjs;
|
|
24
30
|
}
|
|
25
31
|
/**
|
|
26
32
|
* Loads the contents of the PDF as documents.
|
|
27
33
|
* @returns An array of Documents representing the retrieved data.
|
|
28
34
|
*/
|
|
29
35
|
async load() {
|
|
36
|
+
const { getDocument, version } = await this.pdfjs();
|
|
30
37
|
const parsedPdf = await getDocument({
|
|
31
38
|
data: new Uint8Array(await this.blob.arrayBuffer()),
|
|
32
39
|
useWorkerFetch: false,
|
|
@@ -81,3 +88,14 @@ export class WebPDFLoader extends BaseDocumentLoader {
|
|
|
81
88
|
return documents;
|
|
82
89
|
}
|
|
83
90
|
}
|
|
91
|
+
async function PDFLoaderImports() {
|
|
92
|
+
try {
|
|
93
|
+
const { default: mod } = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
|
|
94
|
+
const { getDocument, version } = mod;
|
|
95
|
+
return { getDocument, version };
|
|
96
|
+
}
|
|
97
|
+
catch (e) {
|
|
98
|
+
console.error(e);
|
|
99
|
+
throw new Error("Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.");
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -5,6 +5,20 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.Neo4jGraph = void 0;
|
|
7
7
|
const neo4j_driver_1 = __importDefault(require("neo4j-driver"));
|
|
8
|
+
/**
|
|
9
|
+
* @security *Security note*: Make sure that the database connection uses credentials
|
|
10
|
+
* that are narrowly-scoped to only include necessary permissions.
|
|
11
|
+
* Failure to do so may result in data corruption or loss, since the calling
|
|
12
|
+
* code may attempt commands that would result in deletion, mutation
|
|
13
|
+
* of data if appropriately prompted or reading sensitive data if such
|
|
14
|
+
* data is present in the database.
|
|
15
|
+
* The best way to guard against such negative outcomes is to (as appropriate)
|
|
16
|
+
* limit the permissions granted to the credentials used with this tool.
|
|
17
|
+
* For example, creating read only users for the database is a good way to
|
|
18
|
+
* ensure that the calling code cannot mutate or delete data.
|
|
19
|
+
*
|
|
20
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
21
|
+
*/
|
|
8
22
|
class Neo4jGraph {
|
|
9
23
|
constructor({ url, username, password, database = "neo4j", }) {
|
|
10
24
|
Object.defineProperty(this, "driver", {
|
|
@@ -4,6 +4,20 @@ interface Neo4jGraphConfig {
|
|
|
4
4
|
password: string;
|
|
5
5
|
database?: string;
|
|
6
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* @security *Security note*: Make sure that the database connection uses credentials
|
|
9
|
+
* that are narrowly-scoped to only include necessary permissions.
|
|
10
|
+
* Failure to do so may result in data corruption or loss, since the calling
|
|
11
|
+
* code may attempt commands that would result in deletion, mutation
|
|
12
|
+
* of data if appropriately prompted or reading sensitive data if such
|
|
13
|
+
* data is present in the database.
|
|
14
|
+
* The best way to guard against such negative outcomes is to (as appropriate)
|
|
15
|
+
* limit the permissions granted to the credentials used with this tool.
|
|
16
|
+
* For example, creating read only users for the database is a good way to
|
|
17
|
+
* ensure that the calling code cannot mutate or delete data.
|
|
18
|
+
*
|
|
19
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
20
|
+
*/
|
|
7
21
|
export declare class Neo4jGraph {
|
|
8
22
|
private driver;
|
|
9
23
|
private database;
|
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
import neo4j from "neo4j-driver";
|
|
2
|
+
/**
|
|
3
|
+
* @security *Security note*: Make sure that the database connection uses credentials
|
|
4
|
+
* that are narrowly-scoped to only include necessary permissions.
|
|
5
|
+
* Failure to do so may result in data corruption or loss, since the calling
|
|
6
|
+
* code may attempt commands that would result in deletion, mutation
|
|
7
|
+
* of data if appropriately prompted or reading sensitive data if such
|
|
8
|
+
* data is present in the database.
|
|
9
|
+
* The best way to guard against such negative outcomes is to (as appropriate)
|
|
10
|
+
* limit the permissions granted to the credentials used with this tool.
|
|
11
|
+
* For example, creating read only users for the database is a good way to
|
|
12
|
+
* ensure that the calling code cannot mutate or delete data.
|
|
13
|
+
*
|
|
14
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
15
|
+
*/
|
|
2
16
|
export class Neo4jGraph {
|
|
3
17
|
constructor({ url, username, password, database = "neo4j", }) {
|
|
4
18
|
Object.defineProperty(this, "driver", {
|
package/dist/llms/googlepalm.cjs
CHANGED
|
@@ -83,6 +83,9 @@ class GooglePaLM extends base_js_1.LLM {
|
|
|
83
83
|
if (this.topP && this.topP < 0) {
|
|
84
84
|
throw new Error("`topP` must be a positive integer");
|
|
85
85
|
}
|
|
86
|
+
if (this.topP && this.topP > 1) {
|
|
87
|
+
throw new Error("Google PaLM `topP` must in the range of [0,1]");
|
|
88
|
+
}
|
|
86
89
|
this.topK = fields?.topK ?? this.topK;
|
|
87
90
|
if (this.topK && this.topK < 0) {
|
|
88
91
|
throw new Error("`topK` must be a positive integer");
|
package/dist/llms/googlepalm.js
CHANGED
|
@@ -80,6 +80,9 @@ export class GooglePaLM extends LLM {
|
|
|
80
80
|
if (this.topP && this.topP < 0) {
|
|
81
81
|
throw new Error("`topP` must be a positive integer");
|
|
82
82
|
}
|
|
83
|
+
if (this.topP && this.topP > 1) {
|
|
84
|
+
throw new Error("Google PaLM `topP` must in the range of [0,1]");
|
|
85
|
+
}
|
|
83
86
|
this.topK = fields?.topK ?? this.topK;
|
|
84
87
|
if (this.topK && this.topK < 0) {
|
|
85
88
|
throw new Error("`topK` must be a positive integer");
|
|
@@ -112,6 +112,7 @@ exports.optionalImportEntrypoints = [
|
|
|
112
112
|
"langchain/chat_models/googlevertexai",
|
|
113
113
|
"langchain/chat_models/googlevertexai/web",
|
|
114
114
|
"langchain/chat_models/googlepalm",
|
|
115
|
+
"langchain/chat_models/llama_cpp",
|
|
115
116
|
"langchain/sql_db",
|
|
116
117
|
"langchain/callbacks/handlers/llmonitor",
|
|
117
118
|
"langchain/output_parsers/expression",
|
|
@@ -109,6 +109,7 @@ export const optionalImportEntrypoints = [
|
|
|
109
109
|
"langchain/chat_models/googlevertexai",
|
|
110
110
|
"langchain/chat_models/googlevertexai/web",
|
|
111
111
|
"langchain/chat_models/googlepalm",
|
|
112
|
+
"langchain/chat_models/llama_cpp",
|
|
112
113
|
"langchain/sql_db",
|
|
113
114
|
"langchain/callbacks/handlers/llmonitor",
|
|
114
115
|
"langchain/output_parsers/expression",
|
|
@@ -84,6 +84,12 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
|
|
|
84
84
|
writable: true,
|
|
85
85
|
value: void 0
|
|
86
86
|
});
|
|
87
|
+
Object.defineProperty(this, "childDocumentRetriever", {
|
|
88
|
+
enumerable: true,
|
|
89
|
+
configurable: true,
|
|
90
|
+
writable: true,
|
|
91
|
+
value: void 0
|
|
92
|
+
});
|
|
87
93
|
this.vectorstore = fields.vectorstore;
|
|
88
94
|
this.docstore = fields.docstore;
|
|
89
95
|
this.childSplitter = fields.childSplitter;
|
|
@@ -91,9 +97,17 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
|
|
|
91
97
|
this.idKey = fields.idKey ?? this.idKey;
|
|
92
98
|
this.childK = fields.childK;
|
|
93
99
|
this.parentK = fields.parentK;
|
|
100
|
+
this.childDocumentRetriever = fields.childDocumentRetriever;
|
|
94
101
|
}
|
|
95
102
|
async _getRelevantDocuments(query) {
|
|
96
|
-
|
|
103
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
104
|
+
let subDocs = [];
|
|
105
|
+
if (this.childDocumentRetriever) {
|
|
106
|
+
subDocs = await this.childDocumentRetriever.getRelevantDocuments(query);
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
subDocs = await this.vectorstore.similaritySearch(query, this.childK);
|
|
110
|
+
}
|
|
97
111
|
// Maintain order
|
|
98
112
|
const parentDocIds = [];
|
|
99
113
|
for (const doc of subDocs) {
|
|
@@ -109,6 +123,7 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
|
|
|
109
123
|
}
|
|
110
124
|
/**
|
|
111
125
|
* Adds documents to the docstore and vectorstores.
|
|
126
|
+
* If a retriever is provided, it will be used to add documents instead of the vectorstore.
|
|
112
127
|
* @param docs The documents to add
|
|
113
128
|
* @param config.ids Optional list of ids for documents. If provided should be the same
|
|
114
129
|
* length as the list of documents. Can provided if parent documents
|
|
@@ -150,7 +165,12 @@ class ParentDocumentRetriever extends multi_vector_js_1.MultiVectorRetriever {
|
|
|
150
165
|
embeddedDocs.push(...taggedSubDocs);
|
|
151
166
|
fullDocs[parentDocId] = parentDoc;
|
|
152
167
|
}
|
|
153
|
-
|
|
168
|
+
if (this.childDocumentRetriever) {
|
|
169
|
+
await this.childDocumentRetriever.addDocuments(embeddedDocs);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
await this.vectorstore.addDocuments(embeddedDocs);
|
|
173
|
+
}
|
|
154
174
|
if (addToDocstore) {
|
|
155
175
|
await this.docstore.mset(Object.entries(fullDocs));
|
|
156
176
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Document } from "../document.js";
|
|
2
|
-
import { VectorStore } from "../vectorstores/base.js";
|
|
2
|
+
import { VectorStore, VectorStoreRetriever } from "../vectorstores/base.js";
|
|
3
3
|
import { TextSplitter } from "../text_splitter.js";
|
|
4
4
|
import { MultiVectorRetriever, type MultiVectorRetrieverInput } from "./multi_vector.js";
|
|
5
5
|
/**
|
|
@@ -9,6 +9,11 @@ import { MultiVectorRetriever, type MultiVectorRetrieverInput } from "./multi_ve
|
|
|
9
9
|
export type ParentDocumentRetrieverFields = MultiVectorRetrieverInput & {
|
|
10
10
|
childSplitter: TextSplitter;
|
|
11
11
|
parentSplitter?: TextSplitter;
|
|
12
|
+
/**
|
|
13
|
+
* A custom retriever to use when retrieving instead of
|
|
14
|
+
* the `.similaritySearch` method of the vectorstore.
|
|
15
|
+
*/
|
|
16
|
+
childDocumentRetriever?: VectorStoreRetriever<VectorStore>;
|
|
12
17
|
};
|
|
13
18
|
/**
|
|
14
19
|
* A type of document retriever that splits input documents into smaller chunks
|
|
@@ -28,10 +33,12 @@ export declare class ParentDocumentRetriever extends MultiVectorRetriever {
|
|
|
28
33
|
protected idKey: string;
|
|
29
34
|
protected childK?: number;
|
|
30
35
|
protected parentK?: number;
|
|
36
|
+
childDocumentRetriever: VectorStoreRetriever<VectorStore> | undefined;
|
|
31
37
|
constructor(fields: ParentDocumentRetrieverFields);
|
|
32
38
|
_getRelevantDocuments(query: string): Promise<Document[]>;
|
|
33
39
|
/**
|
|
34
40
|
* Adds documents to the docstore and vectorstores.
|
|
41
|
+
* If a retriever is provided, it will be used to add documents instead of the vectorstore.
|
|
35
42
|
* @param docs The documents to add
|
|
36
43
|
* @param config.ids Optional list of ids for documents. If provided should be the same
|
|
37
44
|
* length as the list of documents. Can provided if parent documents
|
|
@@ -58,6 +58,12 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
|
|
|
58
58
|
writable: true,
|
|
59
59
|
value: void 0
|
|
60
60
|
});
|
|
61
|
+
Object.defineProperty(this, "childDocumentRetriever", {
|
|
62
|
+
enumerable: true,
|
|
63
|
+
configurable: true,
|
|
64
|
+
writable: true,
|
|
65
|
+
value: void 0
|
|
66
|
+
});
|
|
61
67
|
this.vectorstore = fields.vectorstore;
|
|
62
68
|
this.docstore = fields.docstore;
|
|
63
69
|
this.childSplitter = fields.childSplitter;
|
|
@@ -65,9 +71,17 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
|
|
|
65
71
|
this.idKey = fields.idKey ?? this.idKey;
|
|
66
72
|
this.childK = fields.childK;
|
|
67
73
|
this.parentK = fields.parentK;
|
|
74
|
+
this.childDocumentRetriever = fields.childDocumentRetriever;
|
|
68
75
|
}
|
|
69
76
|
async _getRelevantDocuments(query) {
|
|
70
|
-
|
|
77
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
78
|
+
let subDocs = [];
|
|
79
|
+
if (this.childDocumentRetriever) {
|
|
80
|
+
subDocs = await this.childDocumentRetriever.getRelevantDocuments(query);
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
subDocs = await this.vectorstore.similaritySearch(query, this.childK);
|
|
84
|
+
}
|
|
71
85
|
// Maintain order
|
|
72
86
|
const parentDocIds = [];
|
|
73
87
|
for (const doc of subDocs) {
|
|
@@ -83,6 +97,7 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
|
|
|
83
97
|
}
|
|
84
98
|
/**
|
|
85
99
|
* Adds documents to the docstore and vectorstores.
|
|
100
|
+
* If a retriever is provided, it will be used to add documents instead of the vectorstore.
|
|
86
101
|
* @param docs The documents to add
|
|
87
102
|
* @param config.ids Optional list of ids for documents. If provided should be the same
|
|
88
103
|
* length as the list of documents. Can provided if parent documents
|
|
@@ -124,7 +139,12 @@ export class ParentDocumentRetriever extends MultiVectorRetriever {
|
|
|
124
139
|
embeddedDocs.push(...taggedSubDocs);
|
|
125
140
|
fullDocs[parentDocId] = parentDoc;
|
|
126
141
|
}
|
|
127
|
-
|
|
142
|
+
if (this.childDocumentRetriever) {
|
|
143
|
+
await this.childDocumentRetriever.addDocuments(embeddedDocs);
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
await this.vectorstore.addDocuments(embeddedDocs);
|
|
147
|
+
}
|
|
128
148
|
if (addToDocstore) {
|
|
129
149
|
await this.docstore.mset(Object.entries(fullDocs));
|
|
130
150
|
}
|
|
@@ -56,7 +56,9 @@ class RunnablePassthrough extends base_js_1.Runnable {
|
|
|
56
56
|
async invoke(input, options) {
|
|
57
57
|
return this._callWithConfig((input) => Promise.resolve(input), input, options);
|
|
58
58
|
}
|
|
59
|
-
static assign(
|
|
59
|
+
static assign(
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
61
|
+
mapping) {
|
|
60
62
|
return new RunnableAssign(new base_js_1.RunnableMap({ steps: mapping }));
|
|
61
63
|
}
|
|
62
64
|
}
|
|
@@ -52,7 +52,9 @@ export class RunnablePassthrough extends Runnable {
|
|
|
52
52
|
async invoke(input, options) {
|
|
53
53
|
return this._callWithConfig((input) => Promise.resolve(input), input, options);
|
|
54
54
|
}
|
|
55
|
-
static assign(
|
|
55
|
+
static assign(
|
|
56
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
57
|
+
mapping) {
|
|
56
58
|
return new RunnableAssign(new RunnableMap({ steps: mapping }));
|
|
57
59
|
}
|
|
58
60
|
}
|
package/dist/sql_db.cjs
CHANGED
|
@@ -14,6 +14,8 @@ const serializable_js_1 = require("./load/serializable.cjs");
|
|
|
14
14
|
* to read and scope to the tables that are needed.
|
|
15
15
|
* Optionally, use the includesTables or ignoreTables class parameters
|
|
16
16
|
* to limit which tables can/cannot be accessed.
|
|
17
|
+
*
|
|
18
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
17
19
|
*/
|
|
18
20
|
class SqlDatabase extends serializable_js_1.Serializable {
|
|
19
21
|
toJSON() {
|
package/dist/sql_db.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ export type { SqlDatabaseDataSourceParams, SqlDatabaseOptionsParams };
|
|
|
13
13
|
* to read and scope to the tables that are needed.
|
|
14
14
|
* Optionally, use the includesTables or ignoreTables class parameters
|
|
15
15
|
* to limit which tables can/cannot be accessed.
|
|
16
|
+
*
|
|
17
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
16
18
|
*/
|
|
17
19
|
export declare class SqlDatabase extends Serializable implements SqlDatabaseOptionsParams, SqlDatabaseDataSourceParams {
|
|
18
20
|
lc_namespace: string[];
|
package/dist/sql_db.js
CHANGED
|
@@ -11,6 +11,8 @@ import { Serializable } from "./load/serializable.js";
|
|
|
11
11
|
* to read and scope to the tables that are needed.
|
|
12
12
|
* Optionally, use the includesTables or ignoreTables class parameters
|
|
13
13
|
* to limit which tables can/cannot be accessed.
|
|
14
|
+
*
|
|
15
|
+
* @link See https://js.langchain.com/docs/security for more information.
|
|
14
16
|
*/
|
|
15
17
|
export class SqlDatabase extends Serializable {
|
|
16
18
|
toJSON() {
|
package/dist/util/stream.cjs
CHANGED
package/dist/util/stream.js
CHANGED
|
@@ -53,6 +53,12 @@ class CassandraStore extends base_js_1.VectorStore {
|
|
|
53
53
|
writable: true,
|
|
54
54
|
value: void 0
|
|
55
55
|
});
|
|
56
|
+
Object.defineProperty(this, "indices", {
|
|
57
|
+
enumerable: true,
|
|
58
|
+
configurable: true,
|
|
59
|
+
writable: true,
|
|
60
|
+
value: void 0
|
|
61
|
+
});
|
|
56
62
|
Object.defineProperty(this, "isInitialized", {
|
|
57
63
|
enumerable: true,
|
|
58
64
|
configurable: true,
|
|
@@ -65,6 +71,7 @@ class CassandraStore extends base_js_1.VectorStore {
|
|
|
65
71
|
this.table = args.table;
|
|
66
72
|
this.primaryKey = args.primaryKey;
|
|
67
73
|
this.metadataColumns = args.metadataColumns;
|
|
74
|
+
this.indices = args.indices;
|
|
68
75
|
}
|
|
69
76
|
/**
|
|
70
77
|
* Method to save vectors to the Cassandra database.
|
|
@@ -94,13 +101,14 @@ class CassandraStore extends base_js_1.VectorStore {
|
|
|
94
101
|
* Method to search for vectors that are similar to a given query vector.
|
|
95
102
|
* @param query The query vector.
|
|
96
103
|
* @param k The number of similar vectors to return.
|
|
104
|
+
* @param filter
|
|
97
105
|
* @returns Promise that resolves with an array of tuples, each containing a Document and a score.
|
|
98
106
|
*/
|
|
99
|
-
async similaritySearchVectorWithScore(query, k) {
|
|
107
|
+
async similaritySearchVectorWithScore(query, k, filter) {
|
|
100
108
|
if (!this.isInitialized) {
|
|
101
109
|
await this.initialize();
|
|
102
110
|
}
|
|
103
|
-
const queryStr = this.buildSearchQuery(query, k);
|
|
111
|
+
const queryStr = this.buildSearchQuery(query, k, filter);
|
|
104
112
|
const queryResultSet = await this.client.execute(queryStr);
|
|
105
113
|
return queryResultSet?.rows.map((row, index) => {
|
|
106
114
|
const textContent = row.text;
|
|
@@ -173,6 +181,11 @@ class CassandraStore extends base_js_1.VectorStore {
|
|
|
173
181
|
await this.client
|
|
174
182
|
.execute(`CREATE CUSTOM INDEX IF NOT EXISTS idx_vector_${this.table}
|
|
175
183
|
ON ${this.keyspace}.${this.table}(vector) USING 'StorageAttachedIndex';`);
|
|
184
|
+
for await (const { name, value } of this.indices) {
|
|
185
|
+
await this.client
|
|
186
|
+
.execute(`CREATE CUSTOM INDEX IF NOT EXISTS idx_${this.table}_${name}
|
|
187
|
+
ON ${this.keyspace}.${this.table} ${value} USING 'StorageAttachedIndex';`);
|
|
188
|
+
}
|
|
176
189
|
this.isInitialized = true;
|
|
177
190
|
}
|
|
178
191
|
/**
|
|
@@ -200,15 +213,23 @@ class CassandraStore extends base_js_1.VectorStore {
|
|
|
200
213
|
}
|
|
201
214
|
return queries;
|
|
202
215
|
}
|
|
216
|
+
buildWhereClause(filter) {
|
|
217
|
+
const whereClause = Object.entries(filter)
|
|
218
|
+
.map(([key, value]) => `${key} = '${value}'`)
|
|
219
|
+
.join(" AND ");
|
|
220
|
+
return `WHERE ${whereClause}`;
|
|
221
|
+
}
|
|
203
222
|
/**
|
|
204
223
|
* Method to build an CQL query for searching for similar vectors in the
|
|
205
224
|
* Cassandra database.
|
|
206
225
|
* @param query The query vector.
|
|
207
226
|
* @param k The number of similar vectors to return.
|
|
227
|
+
* @param filter
|
|
208
228
|
* @returns The CQL query string.
|
|
209
229
|
*/
|
|
210
|
-
buildSearchQuery(query, k) {
|
|
211
|
-
|
|
230
|
+
buildSearchQuery(query, k = 1, filter = undefined) {
|
|
231
|
+
const whereClause = filter ? this.buildWhereClause(filter) : "";
|
|
232
|
+
return `SELECT * FROM ${this.keyspace}.${this.table} ${whereClause} ORDER BY vector ANN OF [${query}] LIMIT ${k}`;
|
|
212
233
|
}
|
|
213
234
|
}
|
|
214
235
|
exports.CassandraStore = CassandraStore;
|
|
@@ -6,12 +6,17 @@ export interface Column {
|
|
|
6
6
|
type: string;
|
|
7
7
|
name: string;
|
|
8
8
|
}
|
|
9
|
+
export interface Index {
|
|
10
|
+
name: string;
|
|
11
|
+
value: string;
|
|
12
|
+
}
|
|
9
13
|
export interface CassandraLibArgs extends DseClientOptions {
|
|
10
14
|
table: string;
|
|
11
15
|
keyspace: string;
|
|
12
16
|
dimensions: number;
|
|
13
17
|
primaryKey: Column;
|
|
14
18
|
metadataColumns: Column[];
|
|
19
|
+
indices: Index[];
|
|
15
20
|
}
|
|
16
21
|
/**
|
|
17
22
|
* Class for interacting with the Cassandra database. It extends the
|
|
@@ -20,12 +25,14 @@ export interface CassandraLibArgs extends DseClientOptions {
|
|
|
20
25
|
* texts or documents.
|
|
21
26
|
*/
|
|
22
27
|
export declare class CassandraStore extends VectorStore {
|
|
28
|
+
FilterType: Record<string, any>;
|
|
23
29
|
private client;
|
|
24
30
|
private readonly dimensions;
|
|
25
31
|
private readonly keyspace;
|
|
26
32
|
private primaryKey;
|
|
27
33
|
private metadataColumns;
|
|
28
34
|
private readonly table;
|
|
35
|
+
private indices;
|
|
29
36
|
private isInitialized;
|
|
30
37
|
_vectorstoreType(): string;
|
|
31
38
|
constructor(embeddings: Embeddings, args: CassandraLibArgs);
|
|
@@ -46,9 +53,10 @@ export declare class CassandraStore extends VectorStore {
|
|
|
46
53
|
* Method to search for vectors that are similar to a given query vector.
|
|
47
54
|
* @param query The query vector.
|
|
48
55
|
* @param k The number of similar vectors to return.
|
|
56
|
+
* @param filter
|
|
49
57
|
* @returns Promise that resolves with an array of tuples, each containing a Document and a score.
|
|
50
58
|
*/
|
|
51
|
-
similaritySearchVectorWithScore(query: number[], k: number): Promise<[Document, number][]>;
|
|
59
|
+
similaritySearchVectorWithScore(query: number[], k: number, filter?: this["FilterType"]): Promise<[Document, number][]>;
|
|
52
60
|
/**
|
|
53
61
|
* Static method to create an instance of CassandraStore from texts.
|
|
54
62
|
* @param texts The texts to use.
|
|
@@ -87,11 +95,13 @@ export declare class CassandraStore extends VectorStore {
|
|
|
87
95
|
* @returns The CQL query string.
|
|
88
96
|
*/
|
|
89
97
|
private buildInsertQuery;
|
|
98
|
+
private buildWhereClause;
|
|
90
99
|
/**
|
|
91
100
|
* Method to build an CQL query for searching for similar vectors in the
|
|
92
101
|
* Cassandra database.
|
|
93
102
|
* @param query The query vector.
|
|
94
103
|
* @param k The number of similar vectors to return.
|
|
104
|
+
* @param filter
|
|
95
105
|
* @returns The CQL query string.
|
|
96
106
|
*/
|
|
97
107
|
private buildSearchQuery;
|