langchain 0.0.159 → 0.0.161
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/embeddings/bedrock.cjs +73 -0
- package/dist/embeddings/bedrock.d.ts +47 -0
- package/dist/embeddings/bedrock.js +69 -0
- package/dist/experimental/hubs/makersuite/googlemakersuitehub.cjs +297 -0
- package/dist/experimental/hubs/makersuite/googlemakersuitehub.d.ts +183 -0
- package/dist/experimental/hubs/makersuite/googlemakersuitehub.js +291 -0
- package/dist/load/import_constants.cjs +2 -0
- package/dist/load/import_constants.js +2 -0
- package/dist/load/import_map.cjs +4 -1
- package/dist/load/import_map.d.ts +3 -0
- package/dist/load/import_map.js +3 -0
- package/dist/retrievers/chaindesk.cjs +71 -0
- package/dist/retrievers/chaindesk.d.ts +18 -0
- package/dist/retrievers/chaindesk.js +67 -0
- package/dist/retrievers/contextual_compression.cjs +2 -2
- package/dist/retrievers/contextual_compression.js +2 -2
- package/dist/retrievers/databerry.cjs +1 -0
- package/dist/retrievers/databerry.d.ts +1 -0
- package/dist/retrievers/databerry.js +1 -0
- package/dist/retrievers/document_compressors/embeddings_filter.cjs +78 -0
- package/dist/retrievers/document_compressors/embeddings_filter.d.ts +39 -0
- package/dist/retrievers/document_compressors/embeddings_filter.js +74 -0
- package/dist/retrievers/document_compressors/index.cjs +33 -1
- package/dist/retrievers/document_compressors/index.d.ts +14 -1
- package/dist/retrievers/document_compressors/index.js +31 -0
- package/dist/retrievers/document_compressors/test/document_compressor.int.test.cjs +28 -0
- package/dist/retrievers/document_compressors/test/document_compressor.int.test.d.ts +1 -0
- package/dist/retrievers/document_compressors/test/document_compressor.int.test.js +26 -0
- package/dist/retrievers/tavily_search_api.cjs +144 -0
- package/dist/retrievers/tavily_search_api.d.ts +38 -0
- package/dist/retrievers/tavily_search_api.js +140 -0
- package/dist/schema/runnable/base.cjs +3 -0
- package/dist/schema/runnable/base.d.ts +1 -0
- package/dist/schema/runnable/base.js +3 -0
- package/dist/text_splitter.cjs +22 -8
- package/dist/text_splitter.d.ts +1 -0
- package/dist/text_splitter.js +22 -8
- package/dist/types/googlevertexai-types.d.ts +7 -5
- package/dist/util/googlevertexai-connection.cjs +35 -27
- package/dist/util/googlevertexai-connection.d.ts +12 -8
- package/dist/util/googlevertexai-connection.js +33 -26
- package/dist/util/googlevertexai-webauth.d.ts +2 -2
- package/dist/vectorstores/googlevertexai.d.ts +4 -4
- package/dist/vectorstores/pinecone.cjs +21 -34
- package/dist/vectorstores/pinecone.d.ts +6 -9
- package/dist/vectorstores/pinecone.js +21 -34
- package/embeddings/bedrock.cjs +1 -0
- package/embeddings/bedrock.d.ts +1 -0
- package/embeddings/bedrock.js +1 -0
- package/experimental/hubs/makersuite/googlemakersuitehub.cjs +1 -0
- package/experimental/hubs/makersuite/googlemakersuitehub.d.ts +1 -0
- package/experimental/hubs/makersuite/googlemakersuitehub.js +1 -0
- package/package.json +48 -3
- package/retrievers/chaindesk.cjs +1 -0
- package/retrievers/chaindesk.d.ts +1 -0
- package/retrievers/chaindesk.js +1 -0
- package/retrievers/document_compressors/embeddings_filter.cjs +1 -0
- package/retrievers/document_compressors/embeddings_filter.d.ts +1 -0
- package/retrievers/document_compressors/embeddings_filter.js +1 -0
- package/retrievers/tavily_search_api.cjs +1 -0
- package/retrievers/tavily_search_api.d.ts +1 -0
- package/retrievers/tavily_search_api.js +1 -0
|
@@ -858,6 +858,9 @@ class RunnableMap extends Runnable {
|
|
|
858
858
|
this.steps[key] = _coerceToRunnable(value);
|
|
859
859
|
}
|
|
860
860
|
}
|
|
861
|
+
static from(steps) {
|
|
862
|
+
return new RunnableMap({ steps });
|
|
863
|
+
}
|
|
861
864
|
async invoke(input, options
|
|
862
865
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
863
866
|
) {
|
|
@@ -287,6 +287,7 @@ export declare class RunnableMap<RunInput> extends Runnable<RunInput, Record<str
|
|
|
287
287
|
constructor(fields: {
|
|
288
288
|
steps: Record<string, RunnableLike<RunInput>>;
|
|
289
289
|
});
|
|
290
|
+
static from<RunInput>(steps: Record<string, RunnableLike<RunInput>>): RunnableMap<RunInput>;
|
|
290
291
|
invoke(input: RunInput, options?: Partial<BaseCallbackConfig>): Promise<Record<string, any>>;
|
|
291
292
|
}
|
|
292
293
|
/**
|
|
@@ -847,6 +847,9 @@ export class RunnableMap extends Runnable {
|
|
|
847
847
|
this.steps[key] = _coerceToRunnable(value);
|
|
848
848
|
}
|
|
849
849
|
}
|
|
850
|
+
static from(steps) {
|
|
851
|
+
return new RunnableMap({ steps });
|
|
852
|
+
}
|
|
850
853
|
async invoke(input, options
|
|
851
854
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
852
855
|
) {
|
package/dist/text_splitter.cjs
CHANGED
|
@@ -76,21 +76,30 @@ class TextSplitter extends document_js_2.BaseDocumentTransformer {
|
|
|
76
76
|
const text = texts[i];
|
|
77
77
|
let lineCounterIndex = 1;
|
|
78
78
|
let prevChunk = null;
|
|
79
|
+
let indexPrevChunk = -1;
|
|
79
80
|
for (const chunk of await this.splitText(text)) {
|
|
80
81
|
let pageContent = chunkHeader;
|
|
81
82
|
// we need to count the \n that are in the text before getting removed by the splitting
|
|
82
|
-
|
|
83
|
-
if (prevChunk) {
|
|
84
|
-
const
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
83
|
+
const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);
|
|
84
|
+
if (prevChunk === null) {
|
|
85
|
+
const newLinesBeforeFirstChunk = this.numberOfNewLines(text, 0, indexChunk);
|
|
86
|
+
lineCounterIndex += newLinesBeforeFirstChunk;
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
const indexEndPrevChunk = indexPrevChunk + (await this.lengthFunction(prevChunk));
|
|
90
|
+
if (indexEndPrevChunk < indexChunk) {
|
|
91
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexEndPrevChunk, indexChunk);
|
|
92
|
+
lineCounterIndex += numberOfIntermediateNewLines;
|
|
93
|
+
}
|
|
94
|
+
else if (indexEndPrevChunk > indexChunk) {
|
|
95
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
|
|
96
|
+
lineCounterIndex -= numberOfIntermediateNewLines;
|
|
97
|
+
}
|
|
88
98
|
if (appendChunkOverlapHeader) {
|
|
89
99
|
pageContent += chunkOverlapHeader;
|
|
90
100
|
}
|
|
91
101
|
}
|
|
92
|
-
|
|
93
|
-
const newLinesCount = (chunk.match(/\n/g) || []).length;
|
|
102
|
+
const newLinesCount = this.numberOfNewLines(chunk);
|
|
94
103
|
const loc = _metadatas[i].loc && typeof _metadatas[i].loc === "object"
|
|
95
104
|
? { ..._metadatas[i].loc }
|
|
96
105
|
: {};
|
|
@@ -109,10 +118,15 @@ class TextSplitter extends document_js_2.BaseDocumentTransformer {
|
|
|
109
118
|
}));
|
|
110
119
|
lineCounterIndex += newLinesCount;
|
|
111
120
|
prevChunk = chunk;
|
|
121
|
+
indexPrevChunk = indexChunk;
|
|
112
122
|
}
|
|
113
123
|
}
|
|
114
124
|
return documents;
|
|
115
125
|
}
|
|
126
|
+
numberOfNewLines(text, start, end) {
|
|
127
|
+
const textSection = text.slice(start, end);
|
|
128
|
+
return (textSection.match(/\n/g) || []).length;
|
|
129
|
+
}
|
|
116
130
|
async splitDocuments(documents, chunkHeaderOptions = {}) {
|
|
117
131
|
const selectedDocuments = documents.filter((doc) => doc.pageContent !== undefined);
|
|
118
132
|
const texts = selectedDocuments.map((doc) => doc.pageContent);
|
package/dist/text_splitter.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export declare abstract class TextSplitter extends BaseDocumentTransformer imple
|
|
|
23
23
|
abstract splitText(text: string): Promise<string[]>;
|
|
24
24
|
protected splitOnSeparator(text: string, separator: string): string[];
|
|
25
25
|
createDocuments(texts: string[], metadatas?: Record<string, any>[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
|
|
26
|
+
private numberOfNewLines;
|
|
26
27
|
splitDocuments(documents: Document[], chunkHeaderOptions?: TextSplitterChunkHeaderOptions): Promise<Document[]>;
|
|
27
28
|
private joinDocs;
|
|
28
29
|
mergeSplits(splits: string[], separator: string): Promise<string[]>;
|
package/dist/text_splitter.js
CHANGED
|
@@ -73,21 +73,30 @@ export class TextSplitter extends BaseDocumentTransformer {
|
|
|
73
73
|
const text = texts[i];
|
|
74
74
|
let lineCounterIndex = 1;
|
|
75
75
|
let prevChunk = null;
|
|
76
|
+
let indexPrevChunk = -1;
|
|
76
77
|
for (const chunk of await this.splitText(text)) {
|
|
77
78
|
let pageContent = chunkHeader;
|
|
78
79
|
// we need to count the \n that are in the text before getting removed by the splitting
|
|
79
|
-
|
|
80
|
-
if (prevChunk) {
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
80
|
+
const indexChunk = text.indexOf(chunk, indexPrevChunk + 1);
|
|
81
|
+
if (prevChunk === null) {
|
|
82
|
+
const newLinesBeforeFirstChunk = this.numberOfNewLines(text, 0, indexChunk);
|
|
83
|
+
lineCounterIndex += newLinesBeforeFirstChunk;
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
const indexEndPrevChunk = indexPrevChunk + (await this.lengthFunction(prevChunk));
|
|
87
|
+
if (indexEndPrevChunk < indexChunk) {
|
|
88
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexEndPrevChunk, indexChunk);
|
|
89
|
+
lineCounterIndex += numberOfIntermediateNewLines;
|
|
90
|
+
}
|
|
91
|
+
else if (indexEndPrevChunk > indexChunk) {
|
|
92
|
+
const numberOfIntermediateNewLines = this.numberOfNewLines(text, indexChunk, indexEndPrevChunk);
|
|
93
|
+
lineCounterIndex -= numberOfIntermediateNewLines;
|
|
94
|
+
}
|
|
85
95
|
if (appendChunkOverlapHeader) {
|
|
86
96
|
pageContent += chunkOverlapHeader;
|
|
87
97
|
}
|
|
88
98
|
}
|
|
89
|
-
|
|
90
|
-
const newLinesCount = (chunk.match(/\n/g) || []).length;
|
|
99
|
+
const newLinesCount = this.numberOfNewLines(chunk);
|
|
91
100
|
const loc = _metadatas[i].loc && typeof _metadatas[i].loc === "object"
|
|
92
101
|
? { ..._metadatas[i].loc }
|
|
93
102
|
: {};
|
|
@@ -106,10 +115,15 @@ export class TextSplitter extends BaseDocumentTransformer {
|
|
|
106
115
|
}));
|
|
107
116
|
lineCounterIndex += newLinesCount;
|
|
108
117
|
prevChunk = chunk;
|
|
118
|
+
indexPrevChunk = indexChunk;
|
|
109
119
|
}
|
|
110
120
|
}
|
|
111
121
|
return documents;
|
|
112
122
|
}
|
|
123
|
+
numberOfNewLines(text, start, end) {
|
|
124
|
+
const textSection = text.slice(start, end);
|
|
125
|
+
return (textSection.match(/\n/g) || []).length;
|
|
126
|
+
}
|
|
113
127
|
async splitDocuments(documents, chunkHeaderOptions = {}) {
|
|
114
128
|
const selectedDocuments = documents.filter((doc) => doc.pageContent !== undefined);
|
|
115
129
|
const texts = selectedDocuments.map((doc) => doc.pageContent);
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { BaseLLMParams } from "../llms/base.js";
|
|
2
|
-
export interface
|
|
2
|
+
export interface GoogleConnectionParams<AuthOptions> {
|
|
3
|
+
authOptions?: AuthOptions;
|
|
4
|
+
}
|
|
5
|
+
export interface GoogleVertexAIConnectionParams<AuthOptions> extends GoogleConnectionParams<AuthOptions> {
|
|
3
6
|
/** Hostname for the API call */
|
|
4
7
|
endpoint?: string;
|
|
5
|
-
authOptions?: AuthOptions;
|
|
6
8
|
/** Region where the LLM is stored */
|
|
7
9
|
location?: string;
|
|
8
10
|
/** The version of the API functions. Part of the path. */
|
|
@@ -40,10 +42,10 @@ export interface GoogleVertexAIModelParams {
|
|
|
40
42
|
}
|
|
41
43
|
export interface GoogleVertexAIBaseLLMInput<AuthOptions> extends BaseLLMParams, GoogleVertexAIConnectionParams<AuthOptions>, GoogleVertexAIModelParams {
|
|
42
44
|
}
|
|
43
|
-
export interface
|
|
45
|
+
export interface GoogleResponse {
|
|
44
46
|
data: any;
|
|
45
47
|
}
|
|
46
|
-
export interface GoogleVertexAIBasePrediction extends
|
|
48
|
+
export interface GoogleVertexAIBasePrediction extends GoogleResponse {
|
|
47
49
|
safetyAttributes?: any;
|
|
48
50
|
}
|
|
49
51
|
export interface GoogleVertexAILLMResponse<PredictionType extends GoogleVertexAIBasePrediction> {
|
|
@@ -51,7 +53,7 @@ export interface GoogleVertexAILLMResponse<PredictionType extends GoogleVertexAI
|
|
|
51
53
|
predictions: PredictionType[];
|
|
52
54
|
};
|
|
53
55
|
}
|
|
54
|
-
export interface
|
|
56
|
+
export interface GoogleAbstractedClient {
|
|
55
57
|
request: (opts: {
|
|
56
58
|
url?: string;
|
|
57
59
|
method?: "GET" | "POST";
|
|
@@ -1,32 +1,14 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.GoogleVertexAILLMConnection = exports.GoogleVertexAIConnection = void 0;
|
|
4
|
-
class
|
|
5
|
-
constructor(
|
|
3
|
+
exports.GoogleVertexAILLMConnection = exports.GoogleVertexAIConnection = exports.GoogleConnection = void 0;
|
|
4
|
+
class GoogleConnection {
|
|
5
|
+
constructor(caller, client) {
|
|
6
6
|
Object.defineProperty(this, "caller", {
|
|
7
7
|
enumerable: true,
|
|
8
8
|
configurable: true,
|
|
9
9
|
writable: true,
|
|
10
10
|
value: void 0
|
|
11
11
|
});
|
|
12
|
-
Object.defineProperty(this, "endpoint", {
|
|
13
|
-
enumerable: true,
|
|
14
|
-
configurable: true,
|
|
15
|
-
writable: true,
|
|
16
|
-
value: "us-central1-aiplatform.googleapis.com"
|
|
17
|
-
});
|
|
18
|
-
Object.defineProperty(this, "location", {
|
|
19
|
-
enumerable: true,
|
|
20
|
-
configurable: true,
|
|
21
|
-
writable: true,
|
|
22
|
-
value: "us-central1"
|
|
23
|
-
});
|
|
24
|
-
Object.defineProperty(this, "apiVersion", {
|
|
25
|
-
enumerable: true,
|
|
26
|
-
configurable: true,
|
|
27
|
-
writable: true,
|
|
28
|
-
value: "v1"
|
|
29
|
-
});
|
|
30
12
|
Object.defineProperty(this, "client", {
|
|
31
13
|
enumerable: true,
|
|
32
14
|
configurable: true,
|
|
@@ -34,14 +16,8 @@ class GoogleVertexAIConnection {
|
|
|
34
16
|
value: void 0
|
|
35
17
|
});
|
|
36
18
|
this.caller = caller;
|
|
37
|
-
this.endpoint = fields?.endpoint ?? this.endpoint;
|
|
38
|
-
this.location = fields?.location ?? this.location;
|
|
39
|
-
this.apiVersion = fields?.apiVersion ?? this.apiVersion;
|
|
40
19
|
this.client = client;
|
|
41
20
|
}
|
|
42
|
-
buildMethod() {
|
|
43
|
-
return "POST";
|
|
44
|
-
}
|
|
45
21
|
async _request(data, options) {
|
|
46
22
|
const url = await this.buildUrl();
|
|
47
23
|
const method = this.buildMethod();
|
|
@@ -64,6 +40,38 @@ class GoogleVertexAIConnection {
|
|
|
64
40
|
}
|
|
65
41
|
}
|
|
66
42
|
}
|
|
43
|
+
exports.GoogleConnection = GoogleConnection;
|
|
44
|
+
class GoogleVertexAIConnection extends GoogleConnection {
|
|
45
|
+
constructor(fields, caller, client) {
|
|
46
|
+
super(caller, client);
|
|
47
|
+
Object.defineProperty(this, "endpoint", {
|
|
48
|
+
enumerable: true,
|
|
49
|
+
configurable: true,
|
|
50
|
+
writable: true,
|
|
51
|
+
value: "us-central1-aiplatform.googleapis.com"
|
|
52
|
+
});
|
|
53
|
+
Object.defineProperty(this, "location", {
|
|
54
|
+
enumerable: true,
|
|
55
|
+
configurable: true,
|
|
56
|
+
writable: true,
|
|
57
|
+
value: "us-central1"
|
|
58
|
+
});
|
|
59
|
+
Object.defineProperty(this, "apiVersion", {
|
|
60
|
+
enumerable: true,
|
|
61
|
+
configurable: true,
|
|
62
|
+
writable: true,
|
|
63
|
+
value: "v1"
|
|
64
|
+
});
|
|
65
|
+
this.caller = caller;
|
|
66
|
+
this.endpoint = fields?.endpoint ?? this.endpoint;
|
|
67
|
+
this.location = fields?.location ?? this.location;
|
|
68
|
+
this.apiVersion = fields?.apiVersion ?? this.apiVersion;
|
|
69
|
+
this.client = client;
|
|
70
|
+
}
|
|
71
|
+
buildMethod() {
|
|
72
|
+
return "POST";
|
|
73
|
+
}
|
|
74
|
+
}
|
|
67
75
|
exports.GoogleVertexAIConnection = GoogleVertexAIConnection;
|
|
68
76
|
class GoogleVertexAILLMConnection extends GoogleVertexAIConnection {
|
|
69
77
|
constructor(fields, caller, client) {
|
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
import { BaseLanguageModelCallOptions } from "../base_language/index.js";
|
|
2
2
|
import { AsyncCaller, AsyncCallerCallOptions } from "./async_caller.js";
|
|
3
|
-
import type { GoogleVertexAIBaseLLMInput, GoogleVertexAIBasePrediction, GoogleVertexAIConnectionParams, GoogleVertexAILLMResponse, GoogleVertexAIModelParams,
|
|
4
|
-
export declare abstract class
|
|
3
|
+
import type { GoogleVertexAIBaseLLMInput, GoogleVertexAIBasePrediction, GoogleVertexAIConnectionParams, GoogleVertexAILLMResponse, GoogleVertexAIModelParams, GoogleResponse, GoogleAbstractedClient } from "../types/googlevertexai-types.js";
|
|
4
|
+
export declare abstract class GoogleConnection<CallOptions extends AsyncCallerCallOptions, ResponseType extends GoogleResponse> {
|
|
5
5
|
caller: AsyncCaller;
|
|
6
|
+
client: GoogleAbstractedClient;
|
|
7
|
+
constructor(caller: AsyncCaller, client: GoogleAbstractedClient);
|
|
8
|
+
abstract buildUrl(): Promise<string>;
|
|
9
|
+
abstract buildMethod(): string;
|
|
10
|
+
_request(data: unknown | undefined, options: CallOptions): Promise<ResponseType>;
|
|
11
|
+
}
|
|
12
|
+
export declare abstract class GoogleVertexAIConnection<CallOptions extends AsyncCallerCallOptions, ResponseType extends GoogleResponse, AuthOptions> extends GoogleConnection<CallOptions, ResponseType> implements GoogleVertexAIConnectionParams<AuthOptions> {
|
|
6
13
|
endpoint: string;
|
|
7
14
|
location: string;
|
|
8
15
|
apiVersion: string;
|
|
9
|
-
client:
|
|
10
|
-
constructor(fields: GoogleVertexAIConnectionParams<AuthOptions> | undefined, caller: AsyncCaller, client: GoogleVertexAIAbstractedClient);
|
|
11
|
-
abstract buildUrl(): Promise<string>;
|
|
16
|
+
constructor(fields: GoogleVertexAIConnectionParams<AuthOptions> | undefined, caller: AsyncCaller, client: GoogleAbstractedClient);
|
|
12
17
|
buildMethod(): string;
|
|
13
|
-
_request(data: unknown | undefined, options: CallOptions): Promise<ResponseType>;
|
|
14
18
|
}
|
|
15
19
|
export declare class GoogleVertexAILLMConnection<CallOptions extends BaseLanguageModelCallOptions, InstanceType, PredictionType extends GoogleVertexAIBasePrediction, AuthOptions> extends GoogleVertexAIConnection<CallOptions, PredictionType, AuthOptions> implements GoogleVertexAIBaseLLMInput<AuthOptions> {
|
|
16
20
|
model: string;
|
|
17
|
-
client:
|
|
18
|
-
constructor(fields: GoogleVertexAIBaseLLMInput<AuthOptions> | undefined, caller: AsyncCaller, client:
|
|
21
|
+
client: GoogleAbstractedClient;
|
|
22
|
+
constructor(fields: GoogleVertexAIBaseLLMInput<AuthOptions> | undefined, caller: AsyncCaller, client: GoogleAbstractedClient);
|
|
19
23
|
buildUrl(): Promise<string>;
|
|
20
24
|
request(instances: InstanceType[], parameters: GoogleVertexAIModelParams, options: CallOptions): Promise<GoogleVertexAILLMResponse<PredictionType>>;
|
|
21
25
|
}
|
|
@@ -1,29 +1,11 @@
|
|
|
1
|
-
export class
|
|
2
|
-
constructor(
|
|
1
|
+
export class GoogleConnection {
|
|
2
|
+
constructor(caller, client) {
|
|
3
3
|
Object.defineProperty(this, "caller", {
|
|
4
4
|
enumerable: true,
|
|
5
5
|
configurable: true,
|
|
6
6
|
writable: true,
|
|
7
7
|
value: void 0
|
|
8
8
|
});
|
|
9
|
-
Object.defineProperty(this, "endpoint", {
|
|
10
|
-
enumerable: true,
|
|
11
|
-
configurable: true,
|
|
12
|
-
writable: true,
|
|
13
|
-
value: "us-central1-aiplatform.googleapis.com"
|
|
14
|
-
});
|
|
15
|
-
Object.defineProperty(this, "location", {
|
|
16
|
-
enumerable: true,
|
|
17
|
-
configurable: true,
|
|
18
|
-
writable: true,
|
|
19
|
-
value: "us-central1"
|
|
20
|
-
});
|
|
21
|
-
Object.defineProperty(this, "apiVersion", {
|
|
22
|
-
enumerable: true,
|
|
23
|
-
configurable: true,
|
|
24
|
-
writable: true,
|
|
25
|
-
value: "v1"
|
|
26
|
-
});
|
|
27
9
|
Object.defineProperty(this, "client", {
|
|
28
10
|
enumerable: true,
|
|
29
11
|
configurable: true,
|
|
@@ -31,14 +13,8 @@ export class GoogleVertexAIConnection {
|
|
|
31
13
|
value: void 0
|
|
32
14
|
});
|
|
33
15
|
this.caller = caller;
|
|
34
|
-
this.endpoint = fields?.endpoint ?? this.endpoint;
|
|
35
|
-
this.location = fields?.location ?? this.location;
|
|
36
|
-
this.apiVersion = fields?.apiVersion ?? this.apiVersion;
|
|
37
16
|
this.client = client;
|
|
38
17
|
}
|
|
39
|
-
buildMethod() {
|
|
40
|
-
return "POST";
|
|
41
|
-
}
|
|
42
18
|
async _request(data, options) {
|
|
43
19
|
const url = await this.buildUrl();
|
|
44
20
|
const method = this.buildMethod();
|
|
@@ -61,6 +37,37 @@ export class GoogleVertexAIConnection {
|
|
|
61
37
|
}
|
|
62
38
|
}
|
|
63
39
|
}
|
|
40
|
+
export class GoogleVertexAIConnection extends GoogleConnection {
|
|
41
|
+
constructor(fields, caller, client) {
|
|
42
|
+
super(caller, client);
|
|
43
|
+
Object.defineProperty(this, "endpoint", {
|
|
44
|
+
enumerable: true,
|
|
45
|
+
configurable: true,
|
|
46
|
+
writable: true,
|
|
47
|
+
value: "us-central1-aiplatform.googleapis.com"
|
|
48
|
+
});
|
|
49
|
+
Object.defineProperty(this, "location", {
|
|
50
|
+
enumerable: true,
|
|
51
|
+
configurable: true,
|
|
52
|
+
writable: true,
|
|
53
|
+
value: "us-central1"
|
|
54
|
+
});
|
|
55
|
+
Object.defineProperty(this, "apiVersion", {
|
|
56
|
+
enumerable: true,
|
|
57
|
+
configurable: true,
|
|
58
|
+
writable: true,
|
|
59
|
+
value: "v1"
|
|
60
|
+
});
|
|
61
|
+
this.caller = caller;
|
|
62
|
+
this.endpoint = fields?.endpoint ?? this.endpoint;
|
|
63
|
+
this.location = fields?.location ?? this.location;
|
|
64
|
+
this.apiVersion = fields?.apiVersion ?? this.apiVersion;
|
|
65
|
+
this.client = client;
|
|
66
|
+
}
|
|
67
|
+
buildMethod() {
|
|
68
|
+
return "POST";
|
|
69
|
+
}
|
|
70
|
+
}
|
|
64
71
|
export class GoogleVertexAILLMConnection extends GoogleVertexAIConnection {
|
|
65
72
|
constructor(fields, caller, client) {
|
|
66
73
|
super(fields, caller, client);
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Credentials } from "web-auth-library/google";
|
|
2
|
-
import type {
|
|
2
|
+
import type { GoogleAbstractedClient } from "../types/googlevertexai-types.js";
|
|
3
3
|
export type WebGoogleAuthOptions = {
|
|
4
4
|
credentials: string | Credentials;
|
|
5
5
|
scope?: string | string[];
|
|
6
6
|
accessToken?: string;
|
|
7
7
|
};
|
|
8
|
-
export declare class WebGoogleAuth implements
|
|
8
|
+
export declare class WebGoogleAuth implements GoogleAbstractedClient {
|
|
9
9
|
options: WebGoogleAuthOptions;
|
|
10
10
|
constructor(options?: WebGoogleAuthOptions);
|
|
11
11
|
getProjectId(): Promise<string>;
|
|
@@ -4,7 +4,7 @@ import { Embeddings } from "../embeddings/base.js";
|
|
|
4
4
|
import { Document, DocumentInput } from "../document.js";
|
|
5
5
|
import { GoogleVertexAIConnection } from "../util/googlevertexai-connection.js";
|
|
6
6
|
import { AsyncCaller, AsyncCallerCallOptions, AsyncCallerParams } from "../util/async_caller.js";
|
|
7
|
-
import { GoogleVertexAIConnectionParams,
|
|
7
|
+
import { GoogleVertexAIConnectionParams, GoogleResponse } from "../types/googlevertexai-types.js";
|
|
8
8
|
import { Docstore } from "../schema/index.js";
|
|
9
9
|
/**
|
|
10
10
|
* Allows us to create IdDocument classes that contain the ID.
|
|
@@ -26,7 +26,7 @@ interface DeployedIndex {
|
|
|
26
26
|
id: string;
|
|
27
27
|
index: string;
|
|
28
28
|
}
|
|
29
|
-
interface IndexEndpointResponse extends
|
|
29
|
+
interface IndexEndpointResponse extends GoogleResponse {
|
|
30
30
|
data: {
|
|
31
31
|
deployedIndexes: DeployedIndex[];
|
|
32
32
|
publicEndpointDomainName: string;
|
|
@@ -49,7 +49,7 @@ export interface MatchingEngineDeleteParams {
|
|
|
49
49
|
interface RemoveDatapointParams extends GoogleVertexAIConnectionParams<GoogleAuthOptions> {
|
|
50
50
|
index: string;
|
|
51
51
|
}
|
|
52
|
-
interface RemoveDatapointResponse extends
|
|
52
|
+
interface RemoveDatapointResponse extends GoogleResponse {
|
|
53
53
|
}
|
|
54
54
|
declare class RemoveDatapointConnection extends GoogleVertexAIConnection<AsyncCallerCallOptions, RemoveDatapointResponse, GoogleAuthOptions> {
|
|
55
55
|
index: string;
|
|
@@ -75,7 +75,7 @@ interface IndexDatapoint {
|
|
|
75
75
|
restricts?: Restriction[];
|
|
76
76
|
crowdingTag?: CrowdingTag;
|
|
77
77
|
}
|
|
78
|
-
interface UpsertDatapointResponse extends
|
|
78
|
+
interface UpsertDatapointResponse extends GoogleResponse {
|
|
79
79
|
}
|
|
80
80
|
declare class UpsertDatapointConnection extends GoogleVertexAIConnection<AsyncCallerCallOptions, UpsertDatapointResponse, GoogleAuthOptions> {
|
|
81
81
|
index: string;
|
|
@@ -27,11 +27,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
27
27
|
};
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
29
|
exports.PineconeStore = void 0;
|
|
30
|
+
/* eslint-disable no-process-env */
|
|
30
31
|
const uuid = __importStar(require("uuid"));
|
|
31
32
|
const flat_1 = __importDefault(require("flat"));
|
|
32
33
|
const base_js_1 = require("./base.cjs");
|
|
33
34
|
const document_js_1 = require("../document.cjs");
|
|
34
|
-
const chunk_js_1 = require("../util/chunk.cjs");
|
|
35
35
|
const async_caller_js_1 = require("../util/async_caller.cjs");
|
|
36
36
|
/**
|
|
37
37
|
* Class that extends the VectorStore class and provides methods to
|
|
@@ -135,16 +135,13 @@ class PineconeStore extends base_js_1.VectorStore {
|
|
|
135
135
|
values,
|
|
136
136
|
};
|
|
137
137
|
});
|
|
138
|
+
const namespace = this.pineconeIndex.namespace(this.namespace ?? "");
|
|
138
139
|
// Pinecone recommends a limit of 100 vectors per upsert request
|
|
139
|
-
const chunkSize =
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
namespace: this.namespace,
|
|
145
|
-
},
|
|
146
|
-
})));
|
|
147
|
-
await Promise.all(batchRequests);
|
|
140
|
+
const chunkSize = 50;
|
|
141
|
+
for (let i = 0; i < pineconeVectors.length; i += chunkSize) {
|
|
142
|
+
const chunk = pineconeVectors.slice(i, i + chunkSize);
|
|
143
|
+
await namespace.upsert(chunk);
|
|
144
|
+
}
|
|
148
145
|
return documentIds;
|
|
149
146
|
}
|
|
150
147
|
/**
|
|
@@ -153,23 +150,17 @@ class PineconeStore extends base_js_1.VectorStore {
|
|
|
153
150
|
* @returns Promise that resolves when the delete operation is complete.
|
|
154
151
|
*/
|
|
155
152
|
async delete(params) {
|
|
156
|
-
const {
|
|
153
|
+
const { deleteAll, ids } = params;
|
|
154
|
+
const namespace = this.pineconeIndex.namespace(this.namespace ?? "");
|
|
157
155
|
if (deleteAll) {
|
|
158
|
-
await
|
|
159
|
-
deleteAll: true,
|
|
160
|
-
namespace,
|
|
161
|
-
...rest,
|
|
162
|
-
});
|
|
156
|
+
await namespace.deleteAll();
|
|
163
157
|
}
|
|
164
158
|
else if (ids) {
|
|
165
159
|
const batchSize = 1000;
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
...rest,
|
|
171
|
-
})));
|
|
172
|
-
await Promise.all(batchRequests);
|
|
160
|
+
for (let i = 0; i < ids.length; i += batchSize) {
|
|
161
|
+
const batchIds = ids.slice(i, i + batchSize);
|
|
162
|
+
await namespace.deleteMany(batchIds);
|
|
163
|
+
}
|
|
173
164
|
}
|
|
174
165
|
else {
|
|
175
166
|
throw new Error("Either ids or delete_all must be provided.");
|
|
@@ -188,14 +179,12 @@ class PineconeStore extends base_js_1.VectorStore {
|
|
|
188
179
|
throw new Error("cannot provide both `filter` and `this.filter`");
|
|
189
180
|
}
|
|
190
181
|
const _filter = filter ?? this.filter;
|
|
191
|
-
const
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
filter: _filter,
|
|
198
|
-
},
|
|
182
|
+
const namespace = this.pineconeIndex.namespace(this.namespace ?? "");
|
|
183
|
+
const results = await namespace.query({
|
|
184
|
+
includeMetadata: true,
|
|
185
|
+
topK: k,
|
|
186
|
+
vector: query,
|
|
187
|
+
filter: _filter,
|
|
199
188
|
});
|
|
200
189
|
const result = [];
|
|
201
190
|
if (results.matches) {
|
|
@@ -229,9 +218,7 @@ class PineconeStore extends base_js_1.VectorStore {
|
|
|
229
218
|
docs.push(newDoc);
|
|
230
219
|
}
|
|
231
220
|
const args = {
|
|
232
|
-
pineconeIndex:
|
|
233
|
-
? dbConfig.pineconeIndex
|
|
234
|
-
: dbConfig.pineconeClient,
|
|
221
|
+
pineconeIndex: dbConfig.pineconeIndex,
|
|
235
222
|
textKey: dbConfig.textKey,
|
|
236
223
|
namespace: dbConfig.namespace,
|
|
237
224
|
};
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
+
import { Index as PineconeIndex } from "@pinecone-database/pinecone";
|
|
1
2
|
import { VectorStore } from "./base.js";
|
|
2
3
|
import { Embeddings } from "../embeddings/base.js";
|
|
3
4
|
import { Document } from "../document.js";
|
|
4
|
-
import { AsyncCaller
|
|
5
|
+
import { AsyncCaller } from "../util/async_caller.js";
|
|
5
6
|
type PineconeMetadata = Record<string, any>;
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
pineconeIndex: VectorOperationsApi;
|
|
7
|
+
export interface PineconeLibArgs {
|
|
8
|
+
pineconeIndex: PineconeIndex;
|
|
9
9
|
textKey?: string;
|
|
10
10
|
namespace?: string;
|
|
11
11
|
filter?: PineconeMetadata;
|
|
@@ -27,7 +27,7 @@ export declare class PineconeStore extends VectorStore {
|
|
|
27
27
|
FilterType: PineconeMetadata;
|
|
28
28
|
textKey: string;
|
|
29
29
|
namespace?: string;
|
|
30
|
-
pineconeIndex:
|
|
30
|
+
pineconeIndex: PineconeIndex;
|
|
31
31
|
filter?: PineconeMetadata;
|
|
32
32
|
caller: AsyncCaller;
|
|
33
33
|
_vectorstoreType(): string;
|
|
@@ -76,10 +76,7 @@ export declare class PineconeStore extends VectorStore {
|
|
|
76
76
|
* @returns Promise that resolves with a new instance of the PineconeStore class.
|
|
77
77
|
*/
|
|
78
78
|
static fromTexts(texts: string[], metadatas: object[] | object, embeddings: Embeddings, dbConfig: {
|
|
79
|
-
|
|
80
|
-
* @deprecated Use pineconeIndex instead
|
|
81
|
-
*/
|
|
82
|
-
pineconeClient: VectorOperationsApi;
|
|
79
|
+
pineconeIndex: PineconeIndex;
|
|
83
80
|
textKey?: string;
|
|
84
81
|
namespace?: string | undefined;
|
|
85
82
|
} | PineconeLibArgs): Promise<PineconeStore>;
|