langchain 0.0.127 → 0.0.128
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/toolkits/conversational_retrieval/tool.cjs +8 -2
- package/dist/agents/toolkits/conversational_retrieval/tool.d.ts +9 -2
- package/dist/agents/toolkits/conversational_retrieval/tool.js +9 -3
- package/dist/chains/query_constructor/ir.d.ts +5 -4
- package/dist/chains/query_constructor/prompt.cjs +3 -1
- package/dist/chains/query_constructor/prompt.d.ts +2 -2
- package/dist/chains/query_constructor/prompt.js +3 -1
- package/dist/chains/question_answering/refine_prompts.cjs +2 -2
- package/dist/chains/question_answering/refine_prompts.d.ts +1 -1
- package/dist/chains/question_answering/refine_prompts.js +2 -2
- package/dist/chains/sql_db/index.cjs +2 -1
- package/dist/chains/sql_db/index.d.ts +1 -1
- package/dist/chains/sql_db/index.js +1 -1
- package/dist/chains/sql_db/sql_db_prompt.cjs +21 -1
- package/dist/chains/sql_db/sql_db_prompt.d.ts +6 -0
- package/dist/chains/sql_db/sql_db_prompt.js +20 -0
- package/dist/chat_models/openai.cjs +8 -0
- package/dist/chat_models/openai.d.ts +1 -0
- package/dist/chat_models/openai.js +8 -0
- package/dist/document_loaders/web/github.cjs +104 -32
- package/dist/document_loaders/web/github.d.ts +46 -3
- package/dist/document_loaders/web/github.js +104 -32
- package/dist/document_transformers/mozilla_readability.cjs +30 -0
- package/dist/document_transformers/mozilla_readability.d.ts +8 -0
- package/dist/document_transformers/mozilla_readability.js +26 -0
- package/dist/llms/base.cjs +3 -1
- package/dist/llms/base.js +3 -1
- package/dist/llms/bedrock.cjs +152 -0
- package/dist/llms/bedrock.d.ts +54 -0
- package/dist/llms/bedrock.js +148 -0
- package/dist/llms/openai-chat.cjs +8 -0
- package/dist/llms/openai-chat.d.ts +1 -0
- package/dist/llms/openai-chat.js +8 -0
- package/dist/llms/openai.cjs +8 -0
- package/dist/llms/openai.d.ts +1 -0
- package/dist/llms/openai.js +8 -0
- package/dist/load/import_constants.cjs +2 -0
- package/dist/load/import_constants.js +2 -0
- package/dist/retrievers/self_query/base.cjs +31 -0
- package/dist/retrievers/self_query/base.d.ts +5 -2
- package/dist/retrievers/self_query/base.js +31 -0
- package/dist/retrievers/self_query/chroma.d.ts +2 -1
- package/dist/retrievers/self_query/functional.cjs +28 -1
- package/dist/retrievers/self_query/functional.d.ts +3 -0
- package/dist/retrievers/self_query/functional.js +28 -1
- package/dist/retrievers/self_query/index.cjs +18 -4
- package/dist/retrievers/self_query/index.d.ts +14 -10
- package/dist/retrievers/self_query/index.js +18 -4
- package/dist/retrievers/self_query/pinecone.d.ts +2 -1
- package/dist/retrievers/self_query/supabase.cjs +30 -0
- package/dist/retrievers/self_query/supabase.d.ts +3 -5
- package/dist/retrievers/self_query/supabase.js +30 -0
- package/dist/retrievers/self_query/utils.cjs +21 -0
- package/dist/retrievers/self_query/utils.d.ts +2 -0
- package/dist/retrievers/self_query/utils.js +16 -0
- package/dist/retrievers/self_query/weaviate.cjs +67 -16
- package/dist/retrievers/self_query/weaviate.d.ts +6 -3
- package/dist/retrievers/self_query/weaviate.js +67 -16
- package/dist/types/openai-types.d.ts +2 -0
- package/dist/util/event-source-parse.cjs +22 -6
- package/dist/util/event-source-parse.js +22 -6
- package/dist/util/ollama.cjs +16 -2
- package/dist/util/ollama.js +16 -2
- package/dist/util/sql_utils.cjs +38 -3
- package/dist/util/sql_utils.js +39 -4
- package/dist/vectorstores/chroma.cjs +5 -6
- package/dist/vectorstores/chroma.js +5 -6
- package/dist/vectorstores/faiss.cjs +41 -1
- package/dist/vectorstores/faiss.d.ts +6 -0
- package/dist/vectorstores/faiss.js +41 -1
- package/dist/vectorstores/supabase.cjs +10 -4
- package/dist/vectorstores/supabase.d.ts +4 -2
- package/dist/vectorstores/supabase.js +10 -4
- package/document_transformers/mozilla_readability.cjs +1 -0
- package/document_transformers/mozilla_readability.d.ts +1 -0
- package/document_transformers/mozilla_readability.js +1 -0
- package/llms/bedrock.cjs +1 -0
- package/llms/bedrock.d.ts +1 -0
- package/llms/bedrock.js +1 -0
- package/package.json +46 -3
package/dist/llms/openai-chat.js
CHANGED
|
@@ -134,6 +134,12 @@ export class OpenAIChat extends LLM {
|
|
|
134
134
|
writable: true,
|
|
135
135
|
value: void 0
|
|
136
136
|
});
|
|
137
|
+
Object.defineProperty(this, "user", {
|
|
138
|
+
enumerable: true,
|
|
139
|
+
configurable: true,
|
|
140
|
+
writable: true,
|
|
141
|
+
value: void 0
|
|
142
|
+
});
|
|
137
143
|
Object.defineProperty(this, "streaming", {
|
|
138
144
|
enumerable: true,
|
|
139
145
|
configurable: true,
|
|
@@ -222,6 +228,7 @@ export class OpenAIChat extends LLM {
|
|
|
222
228
|
this.logitBias = fields?.logitBias;
|
|
223
229
|
this.maxTokens = fields?.maxTokens;
|
|
224
230
|
this.stop = fields?.stop;
|
|
231
|
+
this.user = fields?.user;
|
|
225
232
|
this.streaming = fields?.streaming ?? false;
|
|
226
233
|
if (this.n > 1) {
|
|
227
234
|
throw new Error("Cannot use n > 1 in OpenAIChat LLM. Use ChatOpenAI Chat Model instead.");
|
|
@@ -257,6 +264,7 @@ export class OpenAIChat extends LLM {
|
|
|
257
264
|
logit_bias: this.logitBias,
|
|
258
265
|
max_tokens: this.maxTokens === -1 ? undefined : this.maxTokens,
|
|
259
266
|
stop: options?.stop ?? this.stop,
|
|
267
|
+
user: this.user,
|
|
260
268
|
stream: this.streaming,
|
|
261
269
|
...this.modelKwargs,
|
|
262
270
|
};
|
package/dist/llms/openai.cjs
CHANGED
|
@@ -147,6 +147,12 @@ class OpenAI extends base_js_1.BaseLLM {
|
|
|
147
147
|
writable: true,
|
|
148
148
|
value: void 0
|
|
149
149
|
});
|
|
150
|
+
Object.defineProperty(this, "user", {
|
|
151
|
+
enumerable: true,
|
|
152
|
+
configurable: true,
|
|
153
|
+
writable: true,
|
|
154
|
+
value: void 0
|
|
155
|
+
});
|
|
150
156
|
Object.defineProperty(this, "streaming", {
|
|
151
157
|
enumerable: true,
|
|
152
158
|
configurable: true,
|
|
@@ -236,6 +242,7 @@ class OpenAI extends base_js_1.BaseLLM {
|
|
|
236
242
|
this.bestOf = fields?.bestOf ?? this.bestOf;
|
|
237
243
|
this.logitBias = fields?.logitBias;
|
|
238
244
|
this.stop = fields?.stop;
|
|
245
|
+
this.user = fields?.user;
|
|
239
246
|
this.streaming = fields?.streaming ?? false;
|
|
240
247
|
if (this.streaming && this.bestOf && this.bestOf > 1) {
|
|
241
248
|
throw new Error("Cannot stream results when bestOf > 1");
|
|
@@ -272,6 +279,7 @@ class OpenAI extends base_js_1.BaseLLM {
|
|
|
272
279
|
best_of: this.bestOf,
|
|
273
280
|
logit_bias: this.logitBias,
|
|
274
281
|
stop: options?.stop ?? this.stop,
|
|
282
|
+
user: this.user,
|
|
275
283
|
stream: this.streaming,
|
|
276
284
|
...this.modelKwargs,
|
|
277
285
|
};
|
package/dist/llms/openai.d.ts
CHANGED
package/dist/llms/openai.js
CHANGED
|
@@ -141,6 +141,12 @@ export class OpenAI extends BaseLLM {
|
|
|
141
141
|
writable: true,
|
|
142
142
|
value: void 0
|
|
143
143
|
});
|
|
144
|
+
Object.defineProperty(this, "user", {
|
|
145
|
+
enumerable: true,
|
|
146
|
+
configurable: true,
|
|
147
|
+
writable: true,
|
|
148
|
+
value: void 0
|
|
149
|
+
});
|
|
144
150
|
Object.defineProperty(this, "streaming", {
|
|
145
151
|
enumerable: true,
|
|
146
152
|
configurable: true,
|
|
@@ -230,6 +236,7 @@ export class OpenAI extends BaseLLM {
|
|
|
230
236
|
this.bestOf = fields?.bestOf ?? this.bestOf;
|
|
231
237
|
this.logitBias = fields?.logitBias;
|
|
232
238
|
this.stop = fields?.stop;
|
|
239
|
+
this.user = fields?.user;
|
|
233
240
|
this.streaming = fields?.streaming ?? false;
|
|
234
241
|
if (this.streaming && this.bestOf && this.bestOf > 1) {
|
|
235
242
|
throw new Error("Cannot stream results when bestOf > 1");
|
|
@@ -266,6 +273,7 @@ export class OpenAI extends BaseLLM {
|
|
|
266
273
|
best_of: this.bestOf,
|
|
267
274
|
logit_bias: this.logitBias,
|
|
268
275
|
stop: options?.stop ?? this.stop,
|
|
276
|
+
user: this.user,
|
|
269
277
|
stream: this.streaming,
|
|
270
278
|
...this.modelKwargs,
|
|
271
279
|
};
|
|
@@ -28,6 +28,7 @@ exports.optionalImportEntrypoints = [
|
|
|
28
28
|
"langchain/llms/googlevertexai",
|
|
29
29
|
"langchain/llms/googlepalm",
|
|
30
30
|
"langchain/llms/sagemaker_endpoint",
|
|
31
|
+
"langchain/llms/bedrock",
|
|
31
32
|
"langchain/prompts/load",
|
|
32
33
|
"langchain/vectorstores/analyticdb",
|
|
33
34
|
"langchain/vectorstores/elasticsearch",
|
|
@@ -82,6 +83,7 @@ exports.optionalImportEntrypoints = [
|
|
|
82
83
|
"langchain/document_loaders/fs/notion",
|
|
83
84
|
"langchain/document_loaders/fs/unstructured",
|
|
84
85
|
"langchain/document_transformers/html_to_text",
|
|
86
|
+
"langchain/document_transformers/mozilla_readability",
|
|
85
87
|
"langchain/chat_models/googlevertexai",
|
|
86
88
|
"langchain/chat_models/googlepalm",
|
|
87
89
|
"langchain/sql_db",
|
|
@@ -25,6 +25,7 @@ export const optionalImportEntrypoints = [
|
|
|
25
25
|
"langchain/llms/googlevertexai",
|
|
26
26
|
"langchain/llms/googlepalm",
|
|
27
27
|
"langchain/llms/sagemaker_endpoint",
|
|
28
|
+
"langchain/llms/bedrock",
|
|
28
29
|
"langchain/prompts/load",
|
|
29
30
|
"langchain/vectorstores/analyticdb",
|
|
30
31
|
"langchain/vectorstores/elasticsearch",
|
|
@@ -79,6 +80,7 @@ export const optionalImportEntrypoints = [
|
|
|
79
80
|
"langchain/document_loaders/fs/notion",
|
|
80
81
|
"langchain/document_loaders/fs/unstructured",
|
|
81
82
|
"langchain/document_transformers/html_to_text",
|
|
83
|
+
"langchain/document_transformers/mozilla_readability",
|
|
82
84
|
"langchain/chat_models/googlevertexai",
|
|
83
85
|
"langchain/chat_models/googlepalm",
|
|
84
86
|
"langchain/sql_db",
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.BasicTranslator = exports.BaseTranslator = void 0;
|
|
4
4
|
const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
|
|
5
|
+
const utils_js_1 = require("./utils.cjs");
|
|
5
6
|
class BaseTranslator extends ir_js_1.Visitor {
|
|
6
7
|
}
|
|
7
8
|
exports.BaseTranslator = BaseTranslator;
|
|
@@ -73,5 +74,35 @@ class BasicTranslator extends BaseTranslator {
|
|
|
73
74
|
}
|
|
74
75
|
return nextArg;
|
|
75
76
|
}
|
|
77
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
78
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) && (0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) || mergeType === "replace") {
|
|
82
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
83
|
+
return undefined;
|
|
84
|
+
}
|
|
85
|
+
return generatedFilter;
|
|
86
|
+
}
|
|
87
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
88
|
+
if (mergeType === "and") {
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
return defaultFilter;
|
|
92
|
+
}
|
|
93
|
+
if (mergeType === "and") {
|
|
94
|
+
return {
|
|
95
|
+
$and: [defaultFilter, generatedFilter],
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
else if (mergeType === "or") {
|
|
99
|
+
return {
|
|
100
|
+
$or: [defaultFilter, generatedFilter],
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
throw new Error("Unknown merge type");
|
|
105
|
+
}
|
|
106
|
+
}
|
|
76
107
|
}
|
|
77
108
|
exports.BasicTranslator = BasicTranslator;
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { Comparator, Comparison, Operation, Operator, StructuredQuery, Visitor, VisitorComparisonResult, VisitorOperationResult, VisitorStructuredQueryResult } from "../../chains/query_constructor/ir.js";
|
|
2
|
+
import { VectorStore } from "../../vectorstores/base.js";
|
|
2
3
|
export type TranslatorOpts = {
|
|
3
4
|
allowedOperators: Operator[];
|
|
4
5
|
allowedComparators: Comparator[];
|
|
5
6
|
};
|
|
6
|
-
export declare abstract class BaseTranslator extends Visitor {
|
|
7
|
+
export declare abstract class BaseTranslator<T extends VectorStore = VectorStore> extends Visitor<T> {
|
|
7
8
|
abstract formatFunction(func: Operator | Comparator): string;
|
|
9
|
+
abstract mergeFilters(defaultFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, generatedFilter: this["VisitStructuredQueryOutput"]["filter"] | undefined, mergeType?: "and" | "or" | "replace"): this["VisitStructuredQueryOutput"]["filter"] | undefined;
|
|
8
10
|
}
|
|
9
|
-
export declare class BasicTranslator extends BaseTranslator {
|
|
11
|
+
export declare class BasicTranslator<T extends VectorStore = VectorStore> extends BaseTranslator<T> {
|
|
10
12
|
VisitOperationOutput: VisitorOperationResult;
|
|
11
13
|
VisitComparisonOutput: VisitorComparisonResult;
|
|
12
14
|
VisitStructuredQueryOutput: VisitorStructuredQueryResult;
|
|
@@ -17,4 +19,5 @@ export declare class BasicTranslator extends BaseTranslator {
|
|
|
17
19
|
visitOperation(operation: Operation): this["VisitOperationOutput"];
|
|
18
20
|
visitComparison(comparison: Comparison): this["VisitComparisonOutput"];
|
|
19
21
|
visitStructuredQuery(query: StructuredQuery): this["VisitStructuredQueryOutput"];
|
|
22
|
+
mergeFilters(defaultFilter: VisitorStructuredQueryResult["filter"] | undefined, generatedFilter: VisitorStructuredQueryResult["filter"] | undefined, mergeType?: string): VisitorStructuredQueryResult["filter"] | undefined;
|
|
20
23
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Comparators, Operators, Visitor, } from "../../chains/query_constructor/ir.js";
|
|
2
|
+
import { isFilterEmpty } from "./utils.js";
|
|
2
3
|
export class BaseTranslator extends Visitor {
|
|
3
4
|
}
|
|
4
5
|
export class BasicTranslator extends BaseTranslator {
|
|
@@ -69,4 +70,34 @@ export class BasicTranslator extends BaseTranslator {
|
|
|
69
70
|
}
|
|
70
71
|
return nextArg;
|
|
71
72
|
}
|
|
73
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
74
|
+
if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
if (isFilterEmpty(defaultFilter) || mergeType === "replace") {
|
|
78
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
79
|
+
return undefined;
|
|
80
|
+
}
|
|
81
|
+
return generatedFilter;
|
|
82
|
+
}
|
|
83
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
84
|
+
if (mergeType === "and") {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
return defaultFilter;
|
|
88
|
+
}
|
|
89
|
+
if (mergeType === "and") {
|
|
90
|
+
return {
|
|
91
|
+
$and: [defaultFilter, generatedFilter],
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
else if (mergeType === "or") {
|
|
95
|
+
return {
|
|
96
|
+
$or: [defaultFilter, generatedFilter],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
throw new Error("Unknown merge type");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
72
103
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
+
import { Chroma } from "../../vectorstores/chroma.js";
|
|
1
2
|
import { BasicTranslator } from "./base.js";
|
|
2
|
-
export declare class ChromaTranslator extends BasicTranslator {
|
|
3
|
+
export declare class ChromaTranslator<T extends Chroma> extends BasicTranslator<T> {
|
|
3
4
|
constructor();
|
|
4
5
|
}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.FunctionalTranslator = void 0;
|
|
4
4
|
const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
|
|
5
5
|
const base_js_1 = require("./base.cjs");
|
|
6
|
+
const utils_js_1 = require("./utils.cjs");
|
|
6
7
|
class FunctionalTranslator extends base_js_1.BaseTranslator {
|
|
7
8
|
constructor() {
|
|
8
9
|
super(...arguments);
|
|
@@ -112,7 +113,7 @@ class FunctionalTranslator extends base_js_1.BaseTranslator {
|
|
|
112
113
|
}
|
|
113
114
|
visitStructuredQuery(query) {
|
|
114
115
|
if (!query.filter) {
|
|
115
|
-
return {
|
|
116
|
+
return {};
|
|
116
117
|
}
|
|
117
118
|
const filterFunction = query.filter?.accept(this);
|
|
118
119
|
if (typeof filterFunction !== "function") {
|
|
@@ -120,5 +121,31 @@ class FunctionalTranslator extends base_js_1.BaseTranslator {
|
|
|
120
121
|
}
|
|
121
122
|
return { filter: filterFunction };
|
|
122
123
|
}
|
|
124
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
125
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) && (0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
126
|
+
return undefined;
|
|
127
|
+
}
|
|
128
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) || mergeType === "replace") {
|
|
129
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
return generatedFilter;
|
|
133
|
+
}
|
|
134
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
135
|
+
if (mergeType === "and") {
|
|
136
|
+
return undefined;
|
|
137
|
+
}
|
|
138
|
+
return defaultFilter;
|
|
139
|
+
}
|
|
140
|
+
if (mergeType === "and") {
|
|
141
|
+
return (document) => defaultFilter(document) && generatedFilter(document);
|
|
142
|
+
}
|
|
143
|
+
else if (mergeType === "or") {
|
|
144
|
+
return (document) => defaultFilter(document) || generatedFilter(document);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
throw new Error("Unknown merge type");
|
|
148
|
+
}
|
|
149
|
+
}
|
|
123
150
|
}
|
|
124
151
|
exports.FunctionalTranslator = FunctionalTranslator;
|
|
@@ -15,6 +15,8 @@ export declare class FunctionalTranslator extends BaseTranslator {
|
|
|
15
15
|
VisitComparisonOutput: FunctionFilter;
|
|
16
16
|
VisitStructuredQueryOutput: {
|
|
17
17
|
filter: FunctionFilter;
|
|
18
|
+
} | {
|
|
19
|
+
[k: string]: never;
|
|
18
20
|
};
|
|
19
21
|
allowedOperators: Operator[];
|
|
20
22
|
allowedComparators: Comparator[];
|
|
@@ -24,5 +26,6 @@ export declare class FunctionalTranslator extends BaseTranslator {
|
|
|
24
26
|
visitOperation(operation: Operation): this["VisitOperationOutput"];
|
|
25
27
|
visitComparison(comparison: Comparison): this["VisitComparisonOutput"];
|
|
26
28
|
visitStructuredQuery(query: StructuredQuery): this["VisitStructuredQueryOutput"];
|
|
29
|
+
mergeFilters(defaultFilter: FunctionFilter, generatedFilter: FunctionFilter, mergeType?: string): FunctionFilter | undefined;
|
|
27
30
|
}
|
|
28
31
|
export {};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Comparators, Operators, } from "../../chains/query_constructor/ir.js";
|
|
2
2
|
import { BaseTranslator } from "./base.js";
|
|
3
|
+
import { isFilterEmpty } from "./utils.js";
|
|
3
4
|
export class FunctionalTranslator extends BaseTranslator {
|
|
4
5
|
constructor() {
|
|
5
6
|
super(...arguments);
|
|
@@ -109,7 +110,7 @@ export class FunctionalTranslator extends BaseTranslator {
|
|
|
109
110
|
}
|
|
110
111
|
visitStructuredQuery(query) {
|
|
111
112
|
if (!query.filter) {
|
|
112
|
-
return {
|
|
113
|
+
return {};
|
|
113
114
|
}
|
|
114
115
|
const filterFunction = query.filter?.accept(this);
|
|
115
116
|
if (typeof filterFunction !== "function") {
|
|
@@ -117,4 +118,30 @@ export class FunctionalTranslator extends BaseTranslator {
|
|
|
117
118
|
}
|
|
118
119
|
return { filter: filterFunction };
|
|
119
120
|
}
|
|
121
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
122
|
+
if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {
|
|
123
|
+
return undefined;
|
|
124
|
+
}
|
|
125
|
+
if (isFilterEmpty(defaultFilter) || mergeType === "replace") {
|
|
126
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
127
|
+
return undefined;
|
|
128
|
+
}
|
|
129
|
+
return generatedFilter;
|
|
130
|
+
}
|
|
131
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
132
|
+
if (mergeType === "and") {
|
|
133
|
+
return undefined;
|
|
134
|
+
}
|
|
135
|
+
return defaultFilter;
|
|
136
|
+
}
|
|
137
|
+
if (mergeType === "and") {
|
|
138
|
+
return (document) => defaultFilter(document) && generatedFilter(document);
|
|
139
|
+
}
|
|
140
|
+
else if (mergeType === "or") {
|
|
141
|
+
return (document) => defaultFilter(document) || generatedFilter(document);
|
|
142
|
+
}
|
|
143
|
+
else {
|
|
144
|
+
throw new Error("Unknown merge type");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
120
147
|
}
|
|
@@ -38,6 +38,12 @@ class SelfQueryRetriever extends retriever_js_1.BaseRetriever {
|
|
|
38
38
|
writable: true,
|
|
39
39
|
value: void 0
|
|
40
40
|
});
|
|
41
|
+
Object.defineProperty(this, "useOriginalQuery", {
|
|
42
|
+
enumerable: true,
|
|
43
|
+
configurable: true,
|
|
44
|
+
writable: true,
|
|
45
|
+
value: false
|
|
46
|
+
});
|
|
41
47
|
Object.defineProperty(this, "searchParams", {
|
|
42
48
|
enumerable: true,
|
|
43
49
|
configurable: true,
|
|
@@ -48,18 +54,26 @@ class SelfQueryRetriever extends retriever_js_1.BaseRetriever {
|
|
|
48
54
|
this.llmChain = options.llmChain;
|
|
49
55
|
this.verbose = options.verbose ?? false;
|
|
50
56
|
this.searchParams = options.searchParams ?? this.searchParams;
|
|
57
|
+
this.useOriginalQuery = options.useOriginalQuery ?? this.useOriginalQuery;
|
|
51
58
|
this.structuredQueryTranslator = options.structuredQueryTranslator;
|
|
52
59
|
}
|
|
53
60
|
async _getRelevantDocuments(query, runManager) {
|
|
54
61
|
const { [this.llmChain.outputKey]: output } = await this.llmChain.call({
|
|
55
62
|
[this.llmChain.inputKeys[0]]: query,
|
|
56
63
|
}, runManager?.getChild("llm_chain"));
|
|
57
|
-
const
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
const generatedStructuredQuery = output;
|
|
65
|
+
const nextArg = this.structuredQueryTranslator.visitStructuredQuery(generatedStructuredQuery);
|
|
66
|
+
const filter = this.structuredQueryTranslator.mergeFilters(this.searchParams?.filter, nextArg.filter, this.searchParams?.mergeFiltersOperator);
|
|
67
|
+
const generatedQuery = generatedStructuredQuery.query;
|
|
68
|
+
let myQuery = query;
|
|
69
|
+
if (!this.useOriginalQuery && generatedQuery && generatedQuery.length > 0) {
|
|
70
|
+
myQuery = generatedQuery;
|
|
71
|
+
}
|
|
72
|
+
if (!filter) {
|
|
73
|
+
return [];
|
|
60
74
|
}
|
|
61
75
|
else {
|
|
62
|
-
return this.vectorStore.similaritySearch(
|
|
76
|
+
return this.vectorStore.similaritySearch(myQuery, this.searchParams?.k, filter, runManager?.getChild("vectorstore"));
|
|
63
77
|
}
|
|
64
78
|
}
|
|
65
79
|
static fromLLM(options) {
|
|
@@ -7,27 +7,31 @@ import { FunctionalTranslator } from "./functional.js";
|
|
|
7
7
|
import { BaseTranslator, BasicTranslator } from "./base.js";
|
|
8
8
|
import { CallbackManagerForRetrieverRun } from "../../callbacks/manager.js";
|
|
9
9
|
export { BaseTranslator, BasicTranslator, FunctionalTranslator };
|
|
10
|
-
export interface SelfQueryRetrieverArgs extends BaseRetrieverInput {
|
|
11
|
-
vectorStore:
|
|
12
|
-
structuredQueryTranslator: BaseTranslator
|
|
10
|
+
export interface SelfQueryRetrieverArgs<T extends VectorStore> extends BaseRetrieverInput {
|
|
11
|
+
vectorStore: T;
|
|
12
|
+
structuredQueryTranslator: BaseTranslator<T>;
|
|
13
13
|
llmChain: LLMChain;
|
|
14
14
|
verbose?: boolean;
|
|
15
|
+
useOriginalQuery?: boolean;
|
|
15
16
|
searchParams?: {
|
|
16
17
|
k?: number;
|
|
17
|
-
filter?:
|
|
18
|
+
filter?: T["FilterType"];
|
|
19
|
+
mergeFiltersOperator?: "or" | "and" | "replace";
|
|
18
20
|
};
|
|
19
21
|
}
|
|
20
|
-
export declare class SelfQueryRetriever extends BaseRetriever implements SelfQueryRetrieverArgs {
|
|
22
|
+
export declare class SelfQueryRetriever<T extends VectorStore> extends BaseRetriever implements SelfQueryRetrieverArgs<T> {
|
|
21
23
|
get lc_namespace(): string[];
|
|
22
|
-
vectorStore:
|
|
24
|
+
vectorStore: T;
|
|
23
25
|
llmChain: LLMChain;
|
|
24
26
|
verbose?: boolean;
|
|
25
|
-
structuredQueryTranslator: BaseTranslator
|
|
27
|
+
structuredQueryTranslator: BaseTranslator<T>;
|
|
28
|
+
useOriginalQuery: boolean;
|
|
26
29
|
searchParams?: {
|
|
27
30
|
k?: number;
|
|
28
|
-
filter?:
|
|
31
|
+
filter?: T["FilterType"];
|
|
32
|
+
mergeFiltersOperator?: "or" | "and" | "replace";
|
|
29
33
|
};
|
|
30
|
-
constructor(options: SelfQueryRetrieverArgs);
|
|
34
|
+
constructor(options: SelfQueryRetrieverArgs<T>);
|
|
31
35
|
_getRelevantDocuments(query: string, runManager?: CallbackManagerForRetrieverRun): Promise<Document<Record<string, unknown>>[]>;
|
|
32
|
-
static fromLLM(options: QueryConstructorChainOptions & Omit<SelfQueryRetrieverArgs
|
|
36
|
+
static fromLLM<T extends VectorStore>(options: QueryConstructorChainOptions & Omit<SelfQueryRetrieverArgs<T>, "llmChain">): SelfQueryRetriever<T>;
|
|
33
37
|
}
|
|
@@ -33,6 +33,12 @@ export class SelfQueryRetriever extends BaseRetriever {
|
|
|
33
33
|
writable: true,
|
|
34
34
|
value: void 0
|
|
35
35
|
});
|
|
36
|
+
Object.defineProperty(this, "useOriginalQuery", {
|
|
37
|
+
enumerable: true,
|
|
38
|
+
configurable: true,
|
|
39
|
+
writable: true,
|
|
40
|
+
value: false
|
|
41
|
+
});
|
|
36
42
|
Object.defineProperty(this, "searchParams", {
|
|
37
43
|
enumerable: true,
|
|
38
44
|
configurable: true,
|
|
@@ -43,18 +49,26 @@ export class SelfQueryRetriever extends BaseRetriever {
|
|
|
43
49
|
this.llmChain = options.llmChain;
|
|
44
50
|
this.verbose = options.verbose ?? false;
|
|
45
51
|
this.searchParams = options.searchParams ?? this.searchParams;
|
|
52
|
+
this.useOriginalQuery = options.useOriginalQuery ?? this.useOriginalQuery;
|
|
46
53
|
this.structuredQueryTranslator = options.structuredQueryTranslator;
|
|
47
54
|
}
|
|
48
55
|
async _getRelevantDocuments(query, runManager) {
|
|
49
56
|
const { [this.llmChain.outputKey]: output } = await this.llmChain.call({
|
|
50
57
|
[this.llmChain.inputKeys[0]]: query,
|
|
51
58
|
}, runManager?.getChild("llm_chain"));
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
|
|
59
|
+
const generatedStructuredQuery = output;
|
|
60
|
+
const nextArg = this.structuredQueryTranslator.visitStructuredQuery(generatedStructuredQuery);
|
|
61
|
+
const filter = this.structuredQueryTranslator.mergeFilters(this.searchParams?.filter, nextArg.filter, this.searchParams?.mergeFiltersOperator);
|
|
62
|
+
const generatedQuery = generatedStructuredQuery.query;
|
|
63
|
+
let myQuery = query;
|
|
64
|
+
if (!this.useOriginalQuery && generatedQuery && generatedQuery.length > 0) {
|
|
65
|
+
myQuery = generatedQuery;
|
|
66
|
+
}
|
|
67
|
+
if (!filter) {
|
|
68
|
+
return [];
|
|
55
69
|
}
|
|
56
70
|
else {
|
|
57
|
-
return this.vectorStore.similaritySearch(
|
|
71
|
+
return this.vectorStore.similaritySearch(myQuery, this.searchParams?.k, filter, runManager?.getChild("vectorstore"));
|
|
58
72
|
}
|
|
59
73
|
}
|
|
60
74
|
static fromLLM(options) {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
+
import { PineconeStore } from "../../vectorstores/pinecone.js";
|
|
1
2
|
import { BasicTranslator } from "./base.js";
|
|
2
|
-
export declare class PineconeTranslator extends BasicTranslator {
|
|
3
|
+
export declare class PineconeTranslator<T extends PineconeStore> extends BasicTranslator<T> {
|
|
3
4
|
constructor();
|
|
4
5
|
}
|
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.SupabaseTranslator = void 0;
|
|
4
4
|
const ir_js_1 = require("../../chains/query_constructor/ir.cjs");
|
|
5
5
|
const base_js_1 = require("./base.cjs");
|
|
6
|
+
const utils_js_1 = require("./utils.cjs");
|
|
6
7
|
class SupabaseTranslator extends base_js_1.BaseTranslator {
|
|
7
8
|
constructor() {
|
|
8
9
|
super(...arguments);
|
|
@@ -138,8 +139,37 @@ class SupabaseTranslator extends base_js_1.BaseTranslator {
|
|
|
138
139
|
}
|
|
139
140
|
}
|
|
140
141
|
visitStructuredQuery(query) {
|
|
142
|
+
if (!query.filter) {
|
|
143
|
+
return {};
|
|
144
|
+
}
|
|
141
145
|
const filterFunction = query.filter?.accept(this);
|
|
142
146
|
return { filter: filterFunction ?? {} };
|
|
143
147
|
}
|
|
148
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
149
|
+
if (mergeType === "or") {
|
|
150
|
+
throw new Error("Supabase self-query filter does not support merging two filters with the OR operator");
|
|
151
|
+
}
|
|
152
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) && (0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
if ((0, utils_js_1.isFilterEmpty)(defaultFilter) || mergeType === "replace") {
|
|
156
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
157
|
+
return undefined;
|
|
158
|
+
}
|
|
159
|
+
return generatedFilter;
|
|
160
|
+
}
|
|
161
|
+
if ((0, utils_js_1.isFilterEmpty)(generatedFilter)) {
|
|
162
|
+
if (mergeType === "and") {
|
|
163
|
+
return undefined;
|
|
164
|
+
}
|
|
165
|
+
return defaultFilter;
|
|
166
|
+
}
|
|
167
|
+
if (mergeType === "and") {
|
|
168
|
+
return (rpc) => generatedFilter(defaultFilter(rpc));
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
throw new Error("Unknown merge type");
|
|
172
|
+
}
|
|
173
|
+
}
|
|
144
174
|
}
|
|
145
175
|
exports.SupabaseTranslator = SupabaseTranslator;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Comparator, Comparison, Operation, Operator, StructuredQuery } from "../../chains/query_constructor/ir.js";
|
|
2
|
-
import { SupabaseFilterRPCCall } from "../../vectorstores/supabase.js";
|
|
2
|
+
import { SupabaseFilterRPCCall, SupabaseVectorStore } from "../../vectorstores/supabase.js";
|
|
3
3
|
import { BaseTranslator } from "./base.js";
|
|
4
4
|
type ValueType = {
|
|
5
5
|
eq: string | number;
|
|
@@ -9,12 +9,9 @@ type ValueType = {
|
|
|
9
9
|
gt: string | number;
|
|
10
10
|
gte: string | number;
|
|
11
11
|
};
|
|
12
|
-
export declare class SupabaseTranslator extends BaseTranslator {
|
|
12
|
+
export declare class SupabaseTranslator<T extends SupabaseVectorStore> extends BaseTranslator<T> {
|
|
13
13
|
VisitOperationOutput: SupabaseFilterRPCCall;
|
|
14
14
|
VisitComparisonOutput: SupabaseFilterRPCCall;
|
|
15
|
-
VisitStructuredQueryOutput: {
|
|
16
|
-
filter: SupabaseFilterRPCCall;
|
|
17
|
-
};
|
|
18
15
|
allowedOperators: Operator[];
|
|
19
16
|
allowedComparators: Comparator[];
|
|
20
17
|
formatFunction(): string;
|
|
@@ -25,5 +22,6 @@ export declare class SupabaseTranslator extends BaseTranslator {
|
|
|
25
22
|
visitComparisonAsString(comparison: Comparison): string;
|
|
26
23
|
visitComparison(comparison: Comparison): this["VisitComparisonOutput"];
|
|
27
24
|
visitStructuredQuery(query: StructuredQuery): this["VisitStructuredQueryOutput"];
|
|
25
|
+
mergeFilters(defaultFilter: SupabaseFilterRPCCall | undefined, generatedFilter: SupabaseFilterRPCCall | undefined, mergeType?: string): SupabaseFilterRPCCall | undefined;
|
|
28
26
|
}
|
|
29
27
|
export {};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Comparators, Operators, } from "../../chains/query_constructor/ir.js";
|
|
2
2
|
import { BaseTranslator } from "./base.js";
|
|
3
|
+
import { isFilterEmpty } from "./utils.js";
|
|
3
4
|
export class SupabaseTranslator extends BaseTranslator {
|
|
4
5
|
constructor() {
|
|
5
6
|
super(...arguments);
|
|
@@ -135,7 +136,36 @@ export class SupabaseTranslator extends BaseTranslator {
|
|
|
135
136
|
}
|
|
136
137
|
}
|
|
137
138
|
visitStructuredQuery(query) {
|
|
139
|
+
if (!query.filter) {
|
|
140
|
+
return {};
|
|
141
|
+
}
|
|
138
142
|
const filterFunction = query.filter?.accept(this);
|
|
139
143
|
return { filter: filterFunction ?? {} };
|
|
140
144
|
}
|
|
145
|
+
mergeFilters(defaultFilter, generatedFilter, mergeType = "and") {
|
|
146
|
+
if (mergeType === "or") {
|
|
147
|
+
throw new Error("Supabase self-query filter does not support merging two filters with the OR operator");
|
|
148
|
+
}
|
|
149
|
+
if (isFilterEmpty(defaultFilter) && isFilterEmpty(generatedFilter)) {
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
152
|
+
if (isFilterEmpty(defaultFilter) || mergeType === "replace") {
|
|
153
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
return generatedFilter;
|
|
157
|
+
}
|
|
158
|
+
if (isFilterEmpty(generatedFilter)) {
|
|
159
|
+
if (mergeType === "and") {
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
return defaultFilter;
|
|
163
|
+
}
|
|
164
|
+
if (mergeType === "and") {
|
|
165
|
+
return (rpc) => generatedFilter(defaultFilter(rpc));
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
throw new Error("Unknown merge type");
|
|
169
|
+
}
|
|
170
|
+
}
|
|
141
171
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.isFilterEmpty = exports.isObject = void 0;
|
|
4
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
5
|
+
function isObject(obj) {
|
|
6
|
+
return obj && typeof obj === "object" && !Array.isArray(obj);
|
|
7
|
+
}
|
|
8
|
+
exports.isObject = isObject;
|
|
9
|
+
function isFilterEmpty(filter) {
|
|
10
|
+
if (!filter)
|
|
11
|
+
return true;
|
|
12
|
+
// for Milvus
|
|
13
|
+
if (typeof filter === "string" && filter.length > 0) {
|
|
14
|
+
return false;
|
|
15
|
+
}
|
|
16
|
+
if (typeof filter === "function") {
|
|
17
|
+
return false;
|
|
18
|
+
}
|
|
19
|
+
return isObject(filter) && Object.keys(filter).length === 0;
|
|
20
|
+
}
|
|
21
|
+
exports.isFilterEmpty = isFilterEmpty;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
2
|
+
export function isObject(obj) {
|
|
3
|
+
return obj && typeof obj === "object" && !Array.isArray(obj);
|
|
4
|
+
}
|
|
5
|
+
export function isFilterEmpty(filter) {
|
|
6
|
+
if (!filter)
|
|
7
|
+
return true;
|
|
8
|
+
// for Milvus
|
|
9
|
+
if (typeof filter === "string" && filter.length > 0) {
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
if (typeof filter === "function") {
|
|
13
|
+
return false;
|
|
14
|
+
}
|
|
15
|
+
return isObject(filter) && Object.keys(filter).length === 0;
|
|
16
|
+
}
|