langchain 0.0.75 → 0.0.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/base_language/count_tokens.cjs +5 -21
- package/dist/base_language/count_tokens.d.ts +1 -6
- package/dist/base_language/count_tokens.js +4 -19
- package/dist/base_language/index.cjs +10 -24
- package/dist/base_language/index.d.ts +3 -2
- package/dist/base_language/index.js +11 -25
- package/dist/cache/redis.d.ts +3 -1
- package/dist/callbacks/base.d.ts +4 -4
- package/dist/callbacks/handlers/initialize.cjs +2 -9
- package/dist/callbacks/handlers/initialize.d.ts +1 -1
- package/dist/callbacks/handlers/initialize.js +2 -9
- package/dist/callbacks/handlers/tracer.cjs +11 -22
- package/dist/callbacks/handlers/tracer.d.ts +2 -2
- package/dist/callbacks/handlers/tracer.js +11 -22
- package/dist/callbacks/handlers/tracer_langchain.cjs +52 -91
- package/dist/callbacks/handlers/tracer_langchain.d.ts +12 -9
- package/dist/callbacks/handlers/tracer_langchain.js +52 -68
- package/dist/callbacks/handlers/tracer_langchain_v1.cjs +5 -1
- package/dist/callbacks/handlers/tracer_langchain_v1.js +5 -1
- package/dist/callbacks/manager.cjs +10 -10
- package/dist/callbacks/manager.d.ts +2 -2
- package/dist/callbacks/manager.js +10 -10
- package/dist/chains/question_answering/map_reduce_prompts.cjs +5 -5
- package/dist/chains/question_answering/map_reduce_prompts.d.ts +1 -1
- package/dist/chains/question_answering/map_reduce_prompts.js +1 -1
- package/dist/chains/question_answering/refine_prompts.cjs +5 -5
- package/dist/chains/question_answering/refine_prompts.d.ts +1 -1
- package/dist/chains/question_answering/refine_prompts.js +1 -1
- package/dist/chains/question_answering/stuff_prompts.cjs +2 -2
- package/dist/chains/question_answering/stuff_prompts.d.ts +1 -1
- package/dist/chains/question_answering/stuff_prompts.js +1 -1
- package/dist/chains/sequential_chain.cjs +2 -2
- package/dist/chains/sequential_chain.d.ts +2 -2
- package/dist/chains/sequential_chain.js +2 -2
- package/dist/chains/sql_db/sql_db_prompt.cjs +20 -1
- package/dist/chains/sql_db/sql_db_prompt.d.ts +1 -0
- package/dist/chains/sql_db/sql_db_prompt.js +19 -0
- package/dist/chat_models/base.cjs +17 -1
- package/dist/chat_models/base.d.ts +6 -0
- package/dist/chat_models/base.js +18 -2
- package/dist/chat_models/openai.cjs +2 -2
- package/dist/chat_models/openai.js +2 -2
- package/dist/client/langchainplus.cjs +90 -29
- package/dist/client/langchainplus.d.ts +7 -5
- package/dist/client/langchainplus.js +90 -29
- package/dist/document_loaders/fs/unstructured.cjs +8 -0
- package/dist/document_loaders/fs/unstructured.d.ts +3 -0
- package/dist/document_loaders/fs/unstructured.js +8 -0
- package/dist/document_loaders/web/apify_dataset.cjs +64 -0
- package/dist/document_loaders/web/apify_dataset.d.ts +28 -0
- package/dist/document_loaders/web/apify_dataset.js +60 -0
- package/dist/embeddings/openai.cjs +2 -2
- package/dist/embeddings/openai.js +2 -2
- package/dist/llms/base.cjs +18 -1
- package/dist/llms/base.d.ts +7 -1
- package/dist/llms/base.js +19 -2
- package/dist/llms/openai-chat.cjs +2 -2
- package/dist/llms/openai-chat.js +2 -2
- package/dist/llms/openai.cjs +2 -2
- package/dist/llms/openai.js +2 -2
- package/dist/memory/base.cjs +9 -1
- package/dist/memory/base.d.ts +1 -0
- package/dist/memory/base.js +7 -0
- package/dist/memory/entity_memory.cjs +151 -0
- package/dist/memory/entity_memory.d.ts +35 -0
- package/dist/memory/entity_memory.js +147 -0
- package/dist/memory/index.cjs +5 -1
- package/dist/memory/index.d.ts +2 -0
- package/dist/memory/index.js +2 -0
- package/dist/memory/prompt.cjs +84 -1
- package/dist/memory/prompt.d.ts +6 -0
- package/dist/memory/prompt.js +83 -0
- package/dist/memory/stores/entity/in_memory.cjs +32 -0
- package/dist/memory/stores/entity/in_memory.d.ts +10 -0
- package/dist/memory/stores/entity/in_memory.js +28 -0
- package/dist/prompts/index.cjs +6 -1
- package/dist/prompts/index.d.ts +1 -0
- package/dist/prompts/index.js +1 -0
- package/dist/{chains/prompt_selector.d.ts → prompts/selectors/conditional.d.ts} +4 -4
- package/dist/schema/index.cjs +13 -1
- package/dist/schema/index.d.ts +17 -0
- package/dist/schema/index.js +11 -0
- package/dist/stores/message/dynamodb.cjs +8 -6
- package/dist/stores/message/dynamodb.js +8 -6
- package/dist/stores/message/redis.cjs +5 -3
- package/dist/stores/message/redis.js +5 -3
- package/dist/stores/message/utils.cjs +30 -15
- package/dist/stores/message/utils.d.ts +4 -2
- package/dist/stores/message/utils.js +28 -14
- package/dist/text_splitter.cjs +3 -23
- package/dist/text_splitter.d.ts +1 -3
- package/dist/text_splitter.js +3 -23
- package/dist/tools/webbrowser.cjs +5 -7
- package/dist/tools/webbrowser.js +3 -5
- package/dist/util/env.cjs +39 -7
- package/dist/util/env.d.ts +19 -0
- package/dist/util/env.js +32 -6
- package/dist/util/sql_utils.cjs +18 -0
- package/dist/util/sql_utils.js +19 -1
- package/dist/util/tiktoken.cjs +26 -0
- package/dist/util/tiktoken.d.ts +9 -0
- package/dist/util/tiktoken.js +21 -0
- package/dist/vectorstores/redis.cjs +236 -0
- package/dist/vectorstores/redis.d.ts +80 -0
- package/dist/vectorstores/redis.js +232 -0
- package/document_loaders/web/apify_dataset.cjs +1 -0
- package/document_loaders/web/apify_dataset.d.ts +1 -0
- package/document_loaders/web/apify_dataset.js +1 -0
- package/package.json +25 -5
- package/vectorstores/redis.cjs +1 -0
- package/vectorstores/redis.d.ts +1 -0
- package/vectorstores/redis.js +1 -0
- /package/dist/{chains/prompt_selector.cjs → prompts/selectors/conditional.cjs} +0 -0
- /package/dist/{chains/prompt_selector.js → prompts/selectors/conditional.js} +0 -0
package/dist/prompts/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.checkValidTemplate = exports.renderTemplate = exports.parseTemplate = exports.BaseChatPromptTemplate = exports.MessagesPlaceholder = exports.ChatMessagePromptTemplate = exports.SystemMessagePromptTemplate = exports.AIMessagePromptTemplate = exports.HumanMessagePromptTemplate = exports.ChatPromptTemplate = exports.FewShotPromptTemplate = exports.SemanticSimilarityExampleSelector = exports.LengthBasedExampleSelector = exports.PromptTemplate = exports.BaseStringPromptTemplate = exports.StringPromptValue = exports.BasePromptTemplate = exports.BaseExampleSelector = void 0;
|
|
3
|
+
exports.checkValidTemplate = exports.renderTemplate = exports.parseTemplate = exports.BaseChatPromptTemplate = exports.MessagesPlaceholder = exports.ChatMessagePromptTemplate = exports.SystemMessagePromptTemplate = exports.AIMessagePromptTemplate = exports.HumanMessagePromptTemplate = exports.ChatPromptTemplate = exports.FewShotPromptTemplate = exports.SemanticSimilarityExampleSelector = exports.LengthBasedExampleSelector = exports.isLLM = exports.isChatModel = exports.ConditionalPromptSelector = exports.BasePromptSelector = exports.PromptTemplate = exports.BaseStringPromptTemplate = exports.StringPromptValue = exports.BasePromptTemplate = exports.BaseExampleSelector = void 0;
|
|
4
4
|
var base_js_1 = require("./base.cjs");
|
|
5
5
|
Object.defineProperty(exports, "BaseExampleSelector", { enumerable: true, get: function () { return base_js_1.BaseExampleSelector; } });
|
|
6
6
|
Object.defineProperty(exports, "BasePromptTemplate", { enumerable: true, get: function () { return base_js_1.BasePromptTemplate; } });
|
|
@@ -8,6 +8,11 @@ Object.defineProperty(exports, "StringPromptValue", { enumerable: true, get: fun
|
|
|
8
8
|
Object.defineProperty(exports, "BaseStringPromptTemplate", { enumerable: true, get: function () { return base_js_1.BaseStringPromptTemplate; } });
|
|
9
9
|
var prompt_js_1 = require("./prompt.cjs");
|
|
10
10
|
Object.defineProperty(exports, "PromptTemplate", { enumerable: true, get: function () { return prompt_js_1.PromptTemplate; } });
|
|
11
|
+
var conditional_js_1 = require("./selectors/conditional.cjs");
|
|
12
|
+
Object.defineProperty(exports, "BasePromptSelector", { enumerable: true, get: function () { return conditional_js_1.BasePromptSelector; } });
|
|
13
|
+
Object.defineProperty(exports, "ConditionalPromptSelector", { enumerable: true, get: function () { return conditional_js_1.ConditionalPromptSelector; } });
|
|
14
|
+
Object.defineProperty(exports, "isChatModel", { enumerable: true, get: function () { return conditional_js_1.isChatModel; } });
|
|
15
|
+
Object.defineProperty(exports, "isLLM", { enumerable: true, get: function () { return conditional_js_1.isLLM; } });
|
|
11
16
|
var LengthBasedExampleSelector_js_1 = require("./selectors/LengthBasedExampleSelector.cjs");
|
|
12
17
|
Object.defineProperty(exports, "LengthBasedExampleSelector", { enumerable: true, get: function () { return LengthBasedExampleSelector_js_1.LengthBasedExampleSelector; } });
|
|
13
18
|
var SemanticSimilarityExampleSelector_js_1 = require("./selectors/SemanticSimilarityExampleSelector.cjs");
|
package/dist/prompts/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { BaseExampleSelector, BasePromptTemplate, BasePromptTemplateInput, StringPromptValue, BaseStringPromptTemplate, } from "./base.js";
|
|
2
2
|
export { PromptTemplate, PromptTemplateInput } from "./prompt.js";
|
|
3
|
+
export { BasePromptSelector, ConditionalPromptSelector, isChatModel, isLLM, } from "./selectors/conditional.js";
|
|
3
4
|
export { LengthBasedExampleSelector, LengthBasedExampleSelectorInput, } from "./selectors/LengthBasedExampleSelector.js";
|
|
4
5
|
export { SemanticSimilarityExampleSelector, SemanticSimilarityExampleSelectorInput, } from "./selectors/SemanticSimilarityExampleSelector.js";
|
|
5
6
|
export { FewShotPromptTemplate, FewShotPromptTemplateInput, } from "./few_shot.js";
|
package/dist/prompts/index.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
export { BaseExampleSelector, BasePromptTemplate, StringPromptValue, BaseStringPromptTemplate, } from "./base.js";
|
|
2
2
|
export { PromptTemplate } from "./prompt.js";
|
|
3
|
+
export { BasePromptSelector, ConditionalPromptSelector, isChatModel, isLLM, } from "./selectors/conditional.js";
|
|
3
4
|
export { LengthBasedExampleSelector, } from "./selectors/LengthBasedExampleSelector.js";
|
|
4
5
|
export { SemanticSimilarityExampleSelector, } from "./selectors/SemanticSimilarityExampleSelector.js";
|
|
5
6
|
export { FewShotPromptTemplate, } from "./few_shot.js";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { BaseChatModel } from "
|
|
2
|
-
import { BasePromptTemplate } from "../
|
|
3
|
-
import { BaseLanguageModel } from "
|
|
4
|
-
import { BaseLLM } from "
|
|
1
|
+
import { BaseChatModel } from "../../chat_models/base.js";
|
|
2
|
+
import { BasePromptTemplate } from "../base.js";
|
|
3
|
+
import { BaseLanguageModel } from "../../base_language/index.js";
|
|
4
|
+
import { BaseLLM } from "../../llms/base.js";
|
|
5
5
|
export declare abstract class BasePromptSelector {
|
|
6
6
|
abstract getPrompt(llm: BaseLanguageModel): BasePromptTemplate;
|
|
7
7
|
}
|
package/dist/schema/index.cjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.BaseFileStore = exports.BaseCache = exports.BaseListChatMessageHistory = exports.BaseChatMessageHistory = exports.BaseRetriever = exports.BasePromptValue = exports.ChatMessage = exports.SystemChatMessage = exports.AIChatMessage = exports.HumanChatMessage = exports.BaseChatMessage = exports.RUN_KEY = void 0;
|
|
3
|
+
exports.BaseEntityStore = exports.BaseFileStore = exports.BaseCache = exports.BaseListChatMessageHistory = exports.BaseChatMessageHistory = exports.BaseRetriever = exports.BasePromptValue = exports.ChatMessage = exports.SystemChatMessage = exports.AIChatMessage = exports.HumanChatMessage = exports.BaseChatMessage = exports.RUN_KEY = void 0;
|
|
4
4
|
exports.RUN_KEY = "__run";
|
|
5
5
|
class BaseChatMessage {
|
|
6
6
|
constructor(text) {
|
|
@@ -20,6 +20,15 @@ class BaseChatMessage {
|
|
|
20
20
|
});
|
|
21
21
|
this.text = text;
|
|
22
22
|
}
|
|
23
|
+
toJSON() {
|
|
24
|
+
return {
|
|
25
|
+
type: this._getType(),
|
|
26
|
+
data: {
|
|
27
|
+
content: this.text,
|
|
28
|
+
role: "role" in this ? this.role : undefined,
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
23
32
|
}
|
|
24
33
|
exports.BaseChatMessage = BaseChatMessage;
|
|
25
34
|
class HumanChatMessage extends BaseChatMessage {
|
|
@@ -86,3 +95,6 @@ exports.BaseCache = BaseCache;
|
|
|
86
95
|
class BaseFileStore {
|
|
87
96
|
}
|
|
88
97
|
exports.BaseFileStore = BaseFileStore;
|
|
98
|
+
class BaseEntityStore {
|
|
99
|
+
}
|
|
100
|
+
exports.BaseEntityStore = BaseEntityStore;
|
package/dist/schema/index.d.ts
CHANGED
|
@@ -34,6 +34,15 @@ export type LLMResult = {
|
|
|
34
34
|
*/
|
|
35
35
|
[RUN_KEY]?: Record<string, any>;
|
|
36
36
|
};
|
|
37
|
+
export interface StoredMessageData {
|
|
38
|
+
content: string;
|
|
39
|
+
role: string | undefined;
|
|
40
|
+
additional_kwargs?: Record<string, any>;
|
|
41
|
+
}
|
|
42
|
+
export interface StoredMessage {
|
|
43
|
+
type: string;
|
|
44
|
+
data: StoredMessageData;
|
|
45
|
+
}
|
|
37
46
|
export type MessageType = "human" | "ai" | "generic" | "system";
|
|
38
47
|
export declare abstract class BaseChatMessage {
|
|
39
48
|
/** The text of the message. */
|
|
@@ -43,6 +52,7 @@ export declare abstract class BaseChatMessage {
|
|
|
43
52
|
/** The type of the message. */
|
|
44
53
|
abstract _getType(): MessageType;
|
|
45
54
|
constructor(text: string);
|
|
55
|
+
toJSON(): StoredMessage;
|
|
46
56
|
}
|
|
47
57
|
export declare class HumanChatMessage extends BaseChatMessage {
|
|
48
58
|
_getType(): MessageType;
|
|
@@ -113,3 +123,10 @@ export declare abstract class BaseFileStore {
|
|
|
113
123
|
abstract readFile(path: string): Promise<string>;
|
|
114
124
|
abstract writeFile(path: string, contents: string): Promise<void>;
|
|
115
125
|
}
|
|
126
|
+
export declare abstract class BaseEntityStore {
|
|
127
|
+
abstract get(key: string, defaultValue?: string): Promise<string | undefined>;
|
|
128
|
+
abstract set(key: string, value?: string): Promise<void>;
|
|
129
|
+
abstract delete(key: string): Promise<void>;
|
|
130
|
+
abstract exists(key: string): Promise<boolean>;
|
|
131
|
+
abstract clear(): Promise<void>;
|
|
132
|
+
}
|
package/dist/schema/index.js
CHANGED
|
@@ -17,6 +17,15 @@ export class BaseChatMessage {
|
|
|
17
17
|
});
|
|
18
18
|
this.text = text;
|
|
19
19
|
}
|
|
20
|
+
toJSON() {
|
|
21
|
+
return {
|
|
22
|
+
type: this._getType(),
|
|
23
|
+
data: {
|
|
24
|
+
content: this.text,
|
|
25
|
+
role: "role" in this ? this.role : undefined,
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
|
20
29
|
}
|
|
21
30
|
export class HumanChatMessage extends BaseChatMessage {
|
|
22
31
|
_getType() {
|
|
@@ -72,3 +81,5 @@ export class BaseCache {
|
|
|
72
81
|
}
|
|
73
82
|
export class BaseFileStore {
|
|
74
83
|
}
|
|
84
|
+
export class BaseEntityStore {
|
|
85
|
+
}
|
|
@@ -74,10 +74,12 @@ class DynamoDBChatMessageHistory extends index_js_1.BaseListChatMessageHistory {
|
|
|
74
74
|
const messages = items
|
|
75
75
|
.map((item) => ({
|
|
76
76
|
type: item.M?.type.S,
|
|
77
|
-
|
|
78
|
-
|
|
77
|
+
data: {
|
|
78
|
+
role: item.M?.role?.S,
|
|
79
|
+
content: item.M?.text.S,
|
|
80
|
+
},
|
|
79
81
|
}))
|
|
80
|
-
.filter((x) => x.type !== undefined && x.
|
|
82
|
+
.filter((x) => x.type !== undefined && x.data.content !== undefined);
|
|
81
83
|
return (0, utils_js_1.mapStoredMessagesToChatMessages)(messages);
|
|
82
84
|
}
|
|
83
85
|
async clear() {
|
|
@@ -107,12 +109,12 @@ class DynamoDBChatMessageHistory extends index_js_1.BaseListChatMessageHistory {
|
|
|
107
109
|
S: message.type,
|
|
108
110
|
},
|
|
109
111
|
text: {
|
|
110
|
-
S: message.
|
|
112
|
+
S: message.data.content,
|
|
111
113
|
},
|
|
112
114
|
},
|
|
113
115
|
};
|
|
114
|
-
if (message.role) {
|
|
115
|
-
dynamoSerializedMessage.M.role = { S: message.role };
|
|
116
|
+
if (message.data.role) {
|
|
117
|
+
dynamoSerializedMessage.M.role = { S: message.data.role };
|
|
116
118
|
}
|
|
117
119
|
return dynamoSerializedMessage;
|
|
118
120
|
}),
|
|
@@ -71,10 +71,12 @@ export class DynamoDBChatMessageHistory extends BaseListChatMessageHistory {
|
|
|
71
71
|
const messages = items
|
|
72
72
|
.map((item) => ({
|
|
73
73
|
type: item.M?.type.S,
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
data: {
|
|
75
|
+
role: item.M?.role?.S,
|
|
76
|
+
content: item.M?.text.S,
|
|
77
|
+
},
|
|
76
78
|
}))
|
|
77
|
-
.filter((x) => x.type !== undefined && x.
|
|
79
|
+
.filter((x) => x.type !== undefined && x.data.content !== undefined);
|
|
78
80
|
return mapStoredMessagesToChatMessages(messages);
|
|
79
81
|
}
|
|
80
82
|
async clear() {
|
|
@@ -104,12 +106,12 @@ export class DynamoDBChatMessageHistory extends BaseListChatMessageHistory {
|
|
|
104
106
|
S: message.type,
|
|
105
107
|
},
|
|
106
108
|
text: {
|
|
107
|
-
S: message.
|
|
109
|
+
S: message.data.content,
|
|
108
110
|
},
|
|
109
111
|
},
|
|
110
112
|
};
|
|
111
|
-
if (message.role) {
|
|
112
|
-
dynamoSerializedMessage.M.role = { S: message.role };
|
|
113
|
+
if (message.data.role) {
|
|
114
|
+
dynamoSerializedMessage.M.role = { S: message.data.role };
|
|
113
115
|
}
|
|
114
116
|
return dynamoSerializedMessage;
|
|
115
117
|
}),
|
|
@@ -45,10 +45,12 @@ class RedisChatMessageHistory extends index_js_1.BaseListChatMessageHistory {
|
|
|
45
45
|
const previousMessages = orderedMessages
|
|
46
46
|
.map((item) => ({
|
|
47
47
|
type: item.type,
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
data: {
|
|
49
|
+
role: item.role,
|
|
50
|
+
content: item.text,
|
|
51
|
+
},
|
|
50
52
|
}))
|
|
51
|
-
.filter((x) => x.type !== undefined && x.
|
|
53
|
+
.filter((x) => x.type !== undefined && x.data.content !== undefined);
|
|
52
54
|
return (0, utils_js_1.mapStoredMessagesToChatMessages)(previousMessages);
|
|
53
55
|
}
|
|
54
56
|
async addMessage(message) {
|
|
@@ -42,10 +42,12 @@ export class RedisChatMessageHistory extends BaseListChatMessageHistory {
|
|
|
42
42
|
const previousMessages = orderedMessages
|
|
43
43
|
.map((item) => ({
|
|
44
44
|
type: item.type,
|
|
45
|
-
|
|
46
|
-
|
|
45
|
+
data: {
|
|
46
|
+
role: item.role,
|
|
47
|
+
content: item.text,
|
|
48
|
+
},
|
|
47
49
|
}))
|
|
48
|
-
.filter((x) => x.type !== undefined && x.
|
|
50
|
+
.filter((x) => x.type !== undefined && x.data.content !== undefined);
|
|
49
51
|
return mapStoredMessagesToChatMessages(previousMessages);
|
|
50
52
|
}
|
|
51
53
|
async addMessage(message) {
|
|
@@ -1,31 +1,46 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.mapChatMessagesToStoredMessages = exports.mapStoredMessagesToChatMessages = void 0;
|
|
3
|
+
exports.mapChatMessagesToStoredMessages = exports.mapStoredMessagesToChatMessages = exports.mapV1MessageToStoredMessage = void 0;
|
|
4
4
|
const index_js_1 = require("../../schema/index.cjs");
|
|
5
|
+
function mapV1MessageToStoredMessage(message) {
|
|
6
|
+
// TODO: Remove this mapper when we deprecate the old message format.
|
|
7
|
+
if (message.data !== undefined) {
|
|
8
|
+
return message;
|
|
9
|
+
}
|
|
10
|
+
else {
|
|
11
|
+
const v1Message = message;
|
|
12
|
+
return {
|
|
13
|
+
type: v1Message.type,
|
|
14
|
+
data: {
|
|
15
|
+
content: v1Message.text,
|
|
16
|
+
role: v1Message.role,
|
|
17
|
+
},
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
exports.mapV1MessageToStoredMessage = mapV1MessageToStoredMessage;
|
|
5
22
|
function mapStoredMessagesToChatMessages(messages) {
|
|
6
23
|
return messages.map((message) => {
|
|
7
|
-
|
|
24
|
+
const storedMessage = mapV1MessageToStoredMessage(message);
|
|
25
|
+
switch (storedMessage.type) {
|
|
8
26
|
case "human":
|
|
9
|
-
return new index_js_1.HumanChatMessage(
|
|
27
|
+
return new index_js_1.HumanChatMessage(storedMessage.data.content);
|
|
10
28
|
case "ai":
|
|
11
|
-
return new index_js_1.AIChatMessage(
|
|
29
|
+
return new index_js_1.AIChatMessage(storedMessage.data.content);
|
|
12
30
|
case "system":
|
|
13
|
-
return new index_js_1.SystemChatMessage(
|
|
14
|
-
|
|
15
|
-
if (
|
|
16
|
-
throw new Error("Role must be defined for
|
|
31
|
+
return new index_js_1.SystemChatMessage(storedMessage.data.content);
|
|
32
|
+
case "chat":
|
|
33
|
+
if (storedMessage.data?.additional_kwargs?.role === undefined) {
|
|
34
|
+
throw new Error("Role must be defined for chat messages");
|
|
17
35
|
}
|
|
18
|
-
return new index_js_1.ChatMessage(
|
|
19
|
-
|
|
36
|
+
return new index_js_1.ChatMessage(storedMessage.data.content, storedMessage.data.additional_kwargs.role);
|
|
37
|
+
default:
|
|
38
|
+
throw new Error(`Got unexpected type: ${storedMessage.type}`);
|
|
20
39
|
}
|
|
21
40
|
});
|
|
22
41
|
}
|
|
23
42
|
exports.mapStoredMessagesToChatMessages = mapStoredMessagesToChatMessages;
|
|
24
43
|
function mapChatMessagesToStoredMessages(messages) {
|
|
25
|
-
return messages.map((message) => (
|
|
26
|
-
type: message._getType(),
|
|
27
|
-
role: "role" in message ? message.role : undefined,
|
|
28
|
-
text: message.text,
|
|
29
|
-
}));
|
|
44
|
+
return messages.map((message) => message.toJSON());
|
|
30
45
|
}
|
|
31
46
|
exports.mapChatMessagesToStoredMessages = mapChatMessagesToStoredMessages;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { BaseChatMessage } from "../../schema/index.js";
|
|
2
|
-
|
|
1
|
+
import { BaseChatMessage, StoredMessage } from "../../schema/index.js";
|
|
2
|
+
interface StoredMessageV1 {
|
|
3
3
|
type: string;
|
|
4
4
|
role: string | undefined;
|
|
5
5
|
text: string;
|
|
6
6
|
}
|
|
7
|
+
export declare function mapV1MessageToStoredMessage(message: StoredMessage | StoredMessageV1): StoredMessage;
|
|
7
8
|
export declare function mapStoredMessagesToChatMessages(messages: StoredMessage[]): BaseChatMessage[];
|
|
8
9
|
export declare function mapChatMessagesToStoredMessages(messages: BaseChatMessage[]): StoredMessage[];
|
|
10
|
+
export {};
|
|
@@ -1,26 +1,40 @@
|
|
|
1
1
|
import { AIChatMessage, ChatMessage, HumanChatMessage, SystemChatMessage, } from "../../schema/index.js";
|
|
2
|
+
export function mapV1MessageToStoredMessage(message) {
|
|
3
|
+
// TODO: Remove this mapper when we deprecate the old message format.
|
|
4
|
+
if (message.data !== undefined) {
|
|
5
|
+
return message;
|
|
6
|
+
}
|
|
7
|
+
else {
|
|
8
|
+
const v1Message = message;
|
|
9
|
+
return {
|
|
10
|
+
type: v1Message.type,
|
|
11
|
+
data: {
|
|
12
|
+
content: v1Message.text,
|
|
13
|
+
role: v1Message.role,
|
|
14
|
+
},
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
}
|
|
2
18
|
export function mapStoredMessagesToChatMessages(messages) {
|
|
3
19
|
return messages.map((message) => {
|
|
4
|
-
|
|
20
|
+
const storedMessage = mapV1MessageToStoredMessage(message);
|
|
21
|
+
switch (storedMessage.type) {
|
|
5
22
|
case "human":
|
|
6
|
-
return new HumanChatMessage(
|
|
23
|
+
return new HumanChatMessage(storedMessage.data.content);
|
|
7
24
|
case "ai":
|
|
8
|
-
return new AIChatMessage(
|
|
25
|
+
return new AIChatMessage(storedMessage.data.content);
|
|
9
26
|
case "system":
|
|
10
|
-
return new SystemChatMessage(
|
|
11
|
-
|
|
12
|
-
if (
|
|
13
|
-
throw new Error("Role must be defined for
|
|
27
|
+
return new SystemChatMessage(storedMessage.data.content);
|
|
28
|
+
case "chat":
|
|
29
|
+
if (storedMessage.data?.additional_kwargs?.role === undefined) {
|
|
30
|
+
throw new Error("Role must be defined for chat messages");
|
|
14
31
|
}
|
|
15
|
-
return new ChatMessage(
|
|
16
|
-
|
|
32
|
+
return new ChatMessage(storedMessage.data.content, storedMessage.data.additional_kwargs.role);
|
|
33
|
+
default:
|
|
34
|
+
throw new Error(`Got unexpected type: ${storedMessage.type}`);
|
|
17
35
|
}
|
|
18
36
|
});
|
|
19
37
|
}
|
|
20
38
|
export function mapChatMessagesToStoredMessages(messages) {
|
|
21
|
-
return messages.map((message) => (
|
|
22
|
-
type: message._getType(),
|
|
23
|
-
role: "role" in message ? message.role : undefined,
|
|
24
|
-
text: message.text,
|
|
25
|
-
}));
|
|
39
|
+
return messages.map((message) => message.toJSON());
|
|
26
40
|
}
|
package/dist/text_splitter.cjs
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MarkdownTextSplitter = exports.TokenTextSplitter = exports.RecursiveCharacterTextSplitter = exports.CharacterTextSplitter = exports.TextSplitter = void 0;
|
|
4
4
|
const document_js_1 = require("./document.cjs");
|
|
5
|
+
const tiktoken_js_1 = require("./util/tiktoken.cjs");
|
|
5
6
|
class TextSplitter {
|
|
6
7
|
constructor(fields) {
|
|
7
8
|
Object.defineProperty(this, "chunkSize", {
|
|
@@ -221,48 +222,27 @@ class TokenTextSplitter extends TextSplitter {
|
|
|
221
222
|
writable: true,
|
|
222
223
|
value: void 0
|
|
223
224
|
});
|
|
224
|
-
Object.defineProperty(this, "registry", {
|
|
225
|
-
enumerable: true,
|
|
226
|
-
configurable: true,
|
|
227
|
-
writable: true,
|
|
228
|
-
value: void 0
|
|
229
|
-
});
|
|
230
225
|
this.encodingName = fields?.encodingName ?? "gpt2";
|
|
231
226
|
this.allowedSpecial = fields?.allowedSpecial ?? [];
|
|
232
227
|
this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
|
|
233
228
|
}
|
|
234
229
|
async splitText(text) {
|
|
235
230
|
if (!this.tokenizer) {
|
|
236
|
-
|
|
237
|
-
this.tokenizer = tiktoken.get_encoding(this.encodingName);
|
|
238
|
-
// We need to register a finalizer to free the tokenizer when the
|
|
239
|
-
// splitter is garbage collected.
|
|
240
|
-
this.registry = new FinalizationRegistry((t) => t.free());
|
|
241
|
-
this.registry.register(this, this.tokenizer);
|
|
231
|
+
this.tokenizer = await (0, tiktoken_js_1.getEncoding)(this.encodingName);
|
|
242
232
|
}
|
|
243
233
|
const splits = [];
|
|
244
234
|
const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
|
|
245
235
|
let start_idx = 0;
|
|
246
236
|
let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
|
|
247
237
|
let chunk_ids = input_ids.slice(start_idx, cur_idx);
|
|
248
|
-
const decoder = new TextDecoder();
|
|
249
238
|
while (start_idx < input_ids.length) {
|
|
250
|
-
splits.push(
|
|
239
|
+
splits.push(this.tokenizer.decode(chunk_ids));
|
|
251
240
|
start_idx += this.chunkSize - this.chunkOverlap;
|
|
252
241
|
cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
|
|
253
242
|
chunk_ids = input_ids.slice(start_idx, cur_idx);
|
|
254
243
|
}
|
|
255
244
|
return splits;
|
|
256
245
|
}
|
|
257
|
-
static async imports() {
|
|
258
|
-
try {
|
|
259
|
-
return await import("@dqbd/tiktoken");
|
|
260
|
-
}
|
|
261
|
-
catch (err) {
|
|
262
|
-
console.error(err);
|
|
263
|
-
throw new Error("Please install @dqbd/tiktoken as a dependency with, e.g. `npm install -S @dqbd/tiktoken`");
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
246
|
}
|
|
267
247
|
exports.TokenTextSplitter = TokenTextSplitter;
|
|
268
248
|
class MarkdownTextSplitter extends RecursiveCharacterTextSplitter {
|
package/dist/text_splitter.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type * as tiktoken from "
|
|
1
|
+
import type * as tiktoken from "js-tiktoken";
|
|
2
2
|
import { Document } from "./document.js";
|
|
3
3
|
export interface TextSplitterParams {
|
|
4
4
|
chunkSize: number;
|
|
@@ -43,10 +43,8 @@ export declare class TokenTextSplitter extends TextSplitter implements TokenText
|
|
|
43
43
|
allowedSpecial: "all" | Array<string>;
|
|
44
44
|
disallowedSpecial: "all" | Array<string>;
|
|
45
45
|
private tokenizer;
|
|
46
|
-
private registry;
|
|
47
46
|
constructor(fields?: Partial<TokenTextSplitterParams>);
|
|
48
47
|
splitText(text: string): Promise<string[]>;
|
|
49
|
-
static imports(): Promise<typeof tiktoken>;
|
|
50
48
|
}
|
|
51
49
|
export type MarkdownTextSplitterParams = TextSplitterParams;
|
|
52
50
|
export declare class MarkdownTextSplitter extends RecursiveCharacterTextSplitter implements MarkdownTextSplitterParams {
|
package/dist/text_splitter.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Document } from "./document.js";
|
|
2
|
+
import { getEncoding } from "./util/tiktoken.js";
|
|
2
3
|
export class TextSplitter {
|
|
3
4
|
constructor(fields) {
|
|
4
5
|
Object.defineProperty(this, "chunkSize", {
|
|
@@ -215,48 +216,27 @@ export class TokenTextSplitter extends TextSplitter {
|
|
|
215
216
|
writable: true,
|
|
216
217
|
value: void 0
|
|
217
218
|
});
|
|
218
|
-
Object.defineProperty(this, "registry", {
|
|
219
|
-
enumerable: true,
|
|
220
|
-
configurable: true,
|
|
221
|
-
writable: true,
|
|
222
|
-
value: void 0
|
|
223
|
-
});
|
|
224
219
|
this.encodingName = fields?.encodingName ?? "gpt2";
|
|
225
220
|
this.allowedSpecial = fields?.allowedSpecial ?? [];
|
|
226
221
|
this.disallowedSpecial = fields?.disallowedSpecial ?? "all";
|
|
227
222
|
}
|
|
228
223
|
async splitText(text) {
|
|
229
224
|
if (!this.tokenizer) {
|
|
230
|
-
|
|
231
|
-
this.tokenizer = tiktoken.get_encoding(this.encodingName);
|
|
232
|
-
// We need to register a finalizer to free the tokenizer when the
|
|
233
|
-
// splitter is garbage collected.
|
|
234
|
-
this.registry = new FinalizationRegistry((t) => t.free());
|
|
235
|
-
this.registry.register(this, this.tokenizer);
|
|
225
|
+
this.tokenizer = await getEncoding(this.encodingName);
|
|
236
226
|
}
|
|
237
227
|
const splits = [];
|
|
238
228
|
const input_ids = this.tokenizer.encode(text, this.allowedSpecial, this.disallowedSpecial);
|
|
239
229
|
let start_idx = 0;
|
|
240
230
|
let cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
|
|
241
231
|
let chunk_ids = input_ids.slice(start_idx, cur_idx);
|
|
242
|
-
const decoder = new TextDecoder();
|
|
243
232
|
while (start_idx < input_ids.length) {
|
|
244
|
-
splits.push(
|
|
233
|
+
splits.push(this.tokenizer.decode(chunk_ids));
|
|
245
234
|
start_idx += this.chunkSize - this.chunkOverlap;
|
|
246
235
|
cur_idx = Math.min(start_idx + this.chunkSize, input_ids.length);
|
|
247
236
|
chunk_ids = input_ids.slice(start_idx, cur_idx);
|
|
248
237
|
}
|
|
249
238
|
return splits;
|
|
250
239
|
}
|
|
251
|
-
static async imports() {
|
|
252
|
-
try {
|
|
253
|
-
return await import("@dqbd/tiktoken");
|
|
254
|
-
}
|
|
255
|
-
catch (err) {
|
|
256
|
-
console.error(err);
|
|
257
|
-
throw new Error("Please install @dqbd/tiktoken as a dependency with, e.g. `npm install -S @dqbd/tiktoken`");
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
240
|
}
|
|
261
241
|
export class MarkdownTextSplitter extends RecursiveCharacterTextSplitter {
|
|
262
242
|
constructor(fields) {
|
|
@@ -28,13 +28,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
28
28
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
29
|
exports.WebBrowser = exports.getText = exports.parseInputs = void 0;
|
|
30
30
|
const axios_1 = __importDefault(require("axios"));
|
|
31
|
-
const browser_or_node_1 = require("browser-or-node");
|
|
32
31
|
const cheerio = __importStar(require("cheerio"));
|
|
32
|
+
const env_js_1 = require("../util/env.cjs");
|
|
33
33
|
const text_splitter_js_1 = require("../text_splitter.cjs");
|
|
34
34
|
const memory_js_1 = require("../vectorstores/memory.cjs");
|
|
35
|
-
const base_js_1 = require("../prompts/base.cjs");
|
|
36
35
|
const document_js_1 = require("../document.cjs");
|
|
37
|
-
const
|
|
36
|
+
const base_js_1 = require("./base.cjs");
|
|
38
37
|
const axios_fetch_adapter_js_1 = __importDefault(require("../util/axios-fetch-adapter.cjs"));
|
|
39
38
|
const parseInputs = (inputs) => {
|
|
40
39
|
const [baseUrl, task] = inputs.split(",").map((input) => {
|
|
@@ -134,7 +133,7 @@ const DEFAULT_HEADERS = {
|
|
|
134
133
|
"Upgrade-Insecure-Requests": "1",
|
|
135
134
|
"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/111.0",
|
|
136
135
|
};
|
|
137
|
-
class WebBrowser extends
|
|
136
|
+
class WebBrowser extends base_js_1.Tool {
|
|
138
137
|
constructor({ model, headers, embeddings, verbose, callbacks, callbackManager, axiosConfig, }) {
|
|
139
138
|
super(verbose, callbacks ?? callbackManager);
|
|
140
139
|
Object.defineProperty(this, "model", {
|
|
@@ -178,7 +177,7 @@ class WebBrowser extends base_js_2.Tool {
|
|
|
178
177
|
this.headers = headers || DEFAULT_HEADERS;
|
|
179
178
|
this.axiosConfig = {
|
|
180
179
|
withCredentials: true,
|
|
181
|
-
adapter:
|
|
180
|
+
adapter: (0, env_js_1.isNode)() ? undefined : axios_fetch_adapter_js_1.default,
|
|
182
181
|
...axiosConfig,
|
|
183
182
|
};
|
|
184
183
|
}
|
|
@@ -218,8 +217,7 @@ class WebBrowser extends base_js_2.Tool {
|
|
|
218
217
|
context = results.map((res) => res.pageContent).join("\n");
|
|
219
218
|
}
|
|
220
219
|
const input = `Text:${context}\n\nI need ${doSummary ? "a summary" : task} from the above text, also provide up to 5 markdown links from within that would be of interest (always including URL and text). Links should be provided, if present, in markdown syntax as a list under the heading "Relevant Links:".`;
|
|
221
|
-
|
|
222
|
-
return res.generations[0][0].text;
|
|
220
|
+
return this.model.predict(input, undefined, runManager?.getChild());
|
|
223
221
|
}
|
|
224
222
|
}
|
|
225
223
|
exports.WebBrowser = WebBrowser;
|
package/dist/tools/webbrowser.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import axiosMod from "axios";
|
|
2
|
-
import { isNode } from "browser-or-node";
|
|
3
2
|
import * as cheerio from "cheerio";
|
|
3
|
+
import { isNode } from "../util/env.js";
|
|
4
4
|
import { RecursiveCharacterTextSplitter } from "../text_splitter.js";
|
|
5
5
|
import { MemoryVectorStore } from "../vectorstores/memory.js";
|
|
6
|
-
import { StringPromptValue } from "../prompts/base.js";
|
|
7
6
|
import { Document } from "../document.js";
|
|
8
7
|
import { Tool } from "./base.js";
|
|
9
8
|
import fetchAdapter from "../util/axios-fetch-adapter.js";
|
|
@@ -147,7 +146,7 @@ export class WebBrowser extends Tool {
|
|
|
147
146
|
this.headers = headers || DEFAULT_HEADERS;
|
|
148
147
|
this.axiosConfig = {
|
|
149
148
|
withCredentials: true,
|
|
150
|
-
adapter: isNode ? undefined : fetchAdapter,
|
|
149
|
+
adapter: isNode() ? undefined : fetchAdapter,
|
|
151
150
|
...axiosConfig,
|
|
152
151
|
};
|
|
153
152
|
}
|
|
@@ -187,7 +186,6 @@ export class WebBrowser extends Tool {
|
|
|
187
186
|
context = results.map((res) => res.pageContent).join("\n");
|
|
188
187
|
}
|
|
189
188
|
const input = `Text:${context}\n\nI need ${doSummary ? "a summary" : task} from the above text, also provide up to 5 markdown links from within that would be of interest (always including URL and text). Links should be provided, if present, in markdown syntax as a list under the heading "Relevant Links:".`;
|
|
190
|
-
|
|
191
|
-
return res.generations[0][0].text;
|
|
189
|
+
return this.model.predict(input, undefined, runManager?.getChild());
|
|
192
190
|
}
|
|
193
191
|
}
|
package/dist/util/env.cjs
CHANGED
|
@@ -1,22 +1,42 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getEnv = void 0;
|
|
4
|
-
const
|
|
3
|
+
exports.getRuntimeEnvironment = exports.getEnv = exports.isNode = exports.isDeno = exports.isJsDom = exports.isWebWorker = exports.isBrowser = void 0;
|
|
4
|
+
const isBrowser = () => typeof window !== "undefined" && typeof window.document !== "undefined";
|
|
5
|
+
exports.isBrowser = isBrowser;
|
|
6
|
+
const isWebWorker = () => typeof globalThis === "object" &&
|
|
7
|
+
globalThis.constructor &&
|
|
8
|
+
globalThis.constructor.name === "DedicatedWorkerGlobalScope";
|
|
9
|
+
exports.isWebWorker = isWebWorker;
|
|
10
|
+
const isJsDom = () => (typeof window !== "undefined" && window.name === "nodejs") ||
|
|
11
|
+
(typeof navigator !== "undefined" &&
|
|
12
|
+
(navigator.userAgent.includes("Node.js") ||
|
|
13
|
+
navigator.userAgent.includes("jsdom")));
|
|
14
|
+
exports.isJsDom = isJsDom;
|
|
15
|
+
// Supabase Edge Function provides a `Deno` global object
|
|
16
|
+
// without `version` property
|
|
17
|
+
const isDeno = () => typeof Deno !== "undefined";
|
|
18
|
+
exports.isDeno = isDeno;
|
|
19
|
+
// Mark not-as-node if in Supabase Edge Function
|
|
20
|
+
const isNode = () => typeof process !== "undefined" &&
|
|
21
|
+
typeof process.versions !== "undefined" &&
|
|
22
|
+
typeof process.versions.node !== "undefined" &&
|
|
23
|
+
!(0, exports.isDeno)();
|
|
24
|
+
exports.isNode = isNode;
|
|
5
25
|
const getEnv = () => {
|
|
6
26
|
let env;
|
|
7
|
-
if (
|
|
27
|
+
if ((0, exports.isBrowser)()) {
|
|
8
28
|
env = "browser";
|
|
9
29
|
}
|
|
10
|
-
else if (
|
|
30
|
+
else if ((0, exports.isNode)()) {
|
|
11
31
|
env = "node";
|
|
12
32
|
}
|
|
13
|
-
else if (
|
|
33
|
+
else if ((0, exports.isWebWorker)()) {
|
|
14
34
|
env = "webworker";
|
|
15
35
|
}
|
|
16
|
-
else if (
|
|
36
|
+
else if ((0, exports.isJsDom)()) {
|
|
17
37
|
env = "jsdom";
|
|
18
38
|
}
|
|
19
|
-
else if (
|
|
39
|
+
else if ((0, exports.isDeno)()) {
|
|
20
40
|
env = "deno";
|
|
21
41
|
}
|
|
22
42
|
else {
|
|
@@ -25,3 +45,15 @@ const getEnv = () => {
|
|
|
25
45
|
return env;
|
|
26
46
|
};
|
|
27
47
|
exports.getEnv = getEnv;
|
|
48
|
+
let runtimeEnvironment;
|
|
49
|
+
async function getRuntimeEnvironment() {
|
|
50
|
+
if (runtimeEnvironment === undefined) {
|
|
51
|
+
const env = (0, exports.getEnv)();
|
|
52
|
+
runtimeEnvironment = {
|
|
53
|
+
library: "langchain-js",
|
|
54
|
+
runtime: env,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return runtimeEnvironment;
|
|
58
|
+
}
|
|
59
|
+
exports.getRuntimeEnvironment = getRuntimeEnvironment;
|