langchainrb 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +53 -25
- data/lib/langchain/assistants/assistant.rb +199 -0
- data/lib/langchain/assistants/message.rb +58 -0
- data/lib/langchain/assistants/thread.rb +34 -0
- data/lib/langchain/chunker/markdown.rb +39 -0
- data/lib/langchain/conversation/memory.rb +1 -6
- data/lib/langchain/conversation.rb +7 -18
- data/lib/langchain/data.rb +4 -3
- data/lib/langchain/llm/ai21.rb +1 -1
- data/lib/langchain/llm/azure.rb +10 -97
- data/lib/langchain/llm/base.rb +1 -0
- data/lib/langchain/llm/cohere.rb +4 -6
- data/lib/langchain/llm/google_palm.rb +2 -0
- data/lib/langchain/llm/google_vertex_ai.rb +12 -10
- data/lib/langchain/llm/openai.rb +104 -160
- data/lib/langchain/llm/replicate.rb +0 -6
- data/lib/langchain/llm/response/anthropic_response.rb +4 -0
- data/lib/langchain/llm/response/google_palm_response.rb +4 -0
- data/lib/langchain/llm/response/ollama_response.rb +5 -1
- data/lib/langchain/llm/response/openai_response.rb +8 -0
- data/lib/langchain/loader.rb +3 -2
- data/lib/langchain/processors/markdown.rb +17 -0
- data/lib/langchain/tool/base.rb +24 -0
- data/lib/langchain/tool/google_search.rb +1 -4
- data/lib/langchain/utils/token_length/ai21_validator.rb +6 -2
- data/lib/langchain/utils/token_length/base_validator.rb +1 -1
- data/lib/langchain/utils/token_length/cohere_validator.rb +6 -2
- data/lib/langchain/utils/token_length/google_palm_validator.rb +5 -1
- data/lib/langchain/utils/token_length/openai_validator.rb +41 -0
- data/lib/langchain/vectorsearch/base.rb +2 -2
- data/lib/langchain/vectorsearch/epsilla.rb +5 -1
- data/lib/langchain/vectorsearch/pinecone.rb +2 -2
- data/lib/langchain/version.rb +1 -1
- data/lib/langchain.rb +2 -1
- metadata +10 -5
data/lib/langchain/tool/base.rb
CHANGED
@@ -91,6 +91,30 @@ module Langchain::Tool
|
|
91
91
|
new.execute(input: input)
|
92
92
|
end
|
93
93
|
|
94
|
+
# Returns the tool as an OpenAI tool
|
95
|
+
#
|
96
|
+
# @return [Hash] tool as an OpenAI tool
|
97
|
+
def to_openai_tool
|
98
|
+
# TODO: This is hardcoded to def execute(input:) found in each tool, needs to be dynamic.
|
99
|
+
{
|
100
|
+
type: "function",
|
101
|
+
function: {
|
102
|
+
name: name,
|
103
|
+
description: description,
|
104
|
+
parameters: {
|
105
|
+
type: "object",
|
106
|
+
properties: {
|
107
|
+
input: {
|
108
|
+
type: "string",
|
109
|
+
description: "Input to the tool"
|
110
|
+
}
|
111
|
+
},
|
112
|
+
required: ["input"]
|
113
|
+
}
|
114
|
+
}
|
115
|
+
}
|
116
|
+
end
|
117
|
+
|
94
118
|
#
|
95
119
|
# Executes the tool and returns the answer
|
96
120
|
#
|
@@ -17,10 +17,7 @@ module Langchain::Tool
|
|
17
17
|
description <<~DESC
|
18
18
|
A wrapper around SerpApi's Google Search API.
|
19
19
|
|
20
|
-
Useful for when you need to answer questions about current events.
|
21
|
-
Always one of the first options when you need to find information on internet.
|
22
|
-
|
23
|
-
Input should be a search query.
|
20
|
+
Useful for when you need to answer questions about current events. Always one of the first options when you need to find information on internet. Input should be a search query.
|
24
21
|
DESC
|
25
22
|
|
26
23
|
attr_reader :api_key
|
@@ -22,8 +22,8 @@ module Langchain
|
|
22
22
|
# @param model_name [String] The model name to validate against
|
23
23
|
# @return [Integer] The token length of the text
|
24
24
|
#
|
25
|
-
def self.token_length(text, model_name,
|
26
|
-
res =
|
25
|
+
def self.token_length(text, model_name, options = {})
|
26
|
+
res = options[:llm].tokenize(text)
|
27
27
|
res.dig(:tokens).length
|
28
28
|
end
|
29
29
|
|
@@ -31,6 +31,10 @@ module Langchain
|
|
31
31
|
TOKEN_LIMITS[model_name]
|
32
32
|
end
|
33
33
|
singleton_class.alias_method :completion_token_limit, :token_limit
|
34
|
+
|
35
|
+
def self.token_length_from_messages(messages, model_name, options)
|
36
|
+
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
37
|
+
end
|
34
38
|
end
|
35
39
|
end
|
36
40
|
end
|
@@ -14,7 +14,7 @@ module Langchain
|
|
14
14
|
class BaseValidator
|
15
15
|
def self.validate_max_tokens!(content, model_name, options = {})
|
16
16
|
text_token_length = if content.is_a?(Array)
|
17
|
-
content
|
17
|
+
token_length_from_messages(content, model_name, options)
|
18
18
|
else
|
19
19
|
token_length(content, model_name, options)
|
20
20
|
end
|
@@ -30,8 +30,8 @@ module Langchain
|
|
30
30
|
# @param model_name [String] The model name to validate against
|
31
31
|
# @return [Integer] The token length of the text
|
32
32
|
#
|
33
|
-
def self.token_length(text, model_name,
|
34
|
-
res =
|
33
|
+
def self.token_length(text, model_name, options = {})
|
34
|
+
res = options[:llm].tokenize(text: text)
|
35
35
|
res["tokens"].length
|
36
36
|
end
|
37
37
|
|
@@ -39,6 +39,10 @@ module Langchain
|
|
39
39
|
TOKEN_LIMITS[model_name]
|
40
40
|
end
|
41
41
|
singleton_class.alias_method :completion_token_limit, :token_limit
|
42
|
+
|
43
|
+
def self.token_length_from_messages(messages, model_name, options)
|
44
|
+
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
45
|
+
end
|
42
46
|
end
|
43
47
|
end
|
44
48
|
end
|
@@ -35,7 +35,7 @@ module Langchain
|
|
35
35
|
# @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
|
36
36
|
# @return [Integer] The token length of the text
|
37
37
|
#
|
38
|
-
def self.token_length(text, model_name = "chat-bison-001", options)
|
38
|
+
def self.token_length(text, model_name = "chat-bison-001", options = {})
|
39
39
|
response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
|
40
40
|
|
41
41
|
raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
|
@@ -43,6 +43,10 @@ module Langchain
|
|
43
43
|
response.dig("tokenCount")
|
44
44
|
end
|
45
45
|
|
46
|
+
def self.token_length_from_messages(messages, model_name, options = {})
|
47
|
+
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
48
|
+
end
|
49
|
+
|
46
50
|
def self.token_limit(model_name)
|
47
51
|
TOKEN_LIMITS.dig(model_name, "input_token_limit")
|
48
52
|
end
|
@@ -75,6 +75,47 @@ module Langchain
|
|
75
75
|
max_tokens = super(content, model_name, options)
|
76
76
|
[options[:max_tokens], max_tokens].reject(&:nil?).min
|
77
77
|
end
|
78
|
+
|
79
|
+
# Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
|
80
|
+
# Return the number of tokens used by a list of messages
|
81
|
+
#
|
82
|
+
# @param messages [Array<Hash>] The messages to calculate the token length for
|
83
|
+
# @param model [String] The model name to validate against
|
84
|
+
# @return [Integer] The token length of the messages
|
85
|
+
#
|
86
|
+
def self.token_length_from_messages(messages, model_name, options = {})
|
87
|
+
encoding = Tiktoken.encoding_for_model(model_name)
|
88
|
+
|
89
|
+
if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
|
90
|
+
tokens_per_message = 3
|
91
|
+
tokens_per_name = 1
|
92
|
+
elsif model_name == "gpt-3.5-turbo-0301"
|
93
|
+
tokens_per_message = 4 # every message follows {role/name}\n{content}\n
|
94
|
+
tokens_per_name = -1 # if there's a name, the role is omitted
|
95
|
+
elsif model_name.include?("gpt-3.5-turbo")
|
96
|
+
# puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
|
97
|
+
return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
|
98
|
+
elsif model_name.include?("gpt-4")
|
99
|
+
# puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
|
100
|
+
return token_length_from_messages(messages, "gpt-4-0613", options)
|
101
|
+
else
|
102
|
+
raise NotImplementedError.new(
|
103
|
+
"token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
|
104
|
+
)
|
105
|
+
end
|
106
|
+
|
107
|
+
num_tokens = 0
|
108
|
+
messages.each do |message|
|
109
|
+
num_tokens += tokens_per_message
|
110
|
+
message.each do |key, value|
|
111
|
+
num_tokens += encoding.encode(value).length
|
112
|
+
num_tokens += tokens_per_name if ["name", :name].include?(key)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
num_tokens += 3 # every reply is primed with assistant
|
117
|
+
num_tokens
|
118
|
+
end
|
78
119
|
end
|
79
120
|
end
|
80
121
|
end
|
@@ -175,13 +175,13 @@ module Langchain::Vectorsearch
|
|
175
175
|
prompt_template.format(question: question, context: context)
|
176
176
|
end
|
177
177
|
|
178
|
-
def add_data(paths:)
|
178
|
+
def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
|
179
179
|
raise ArgumentError, "Paths must be provided" if Array(paths).empty?
|
180
180
|
|
181
181
|
texts = Array(paths)
|
182
182
|
.flatten
|
183
183
|
.map do |path|
|
184
|
-
data = Langchain::Loader.new(path)&.load&.chunks
|
184
|
+
data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
|
185
185
|
data.map { |chunk| chunk.text }
|
186
186
|
end
|
187
187
|
|
@@ -36,7 +36,11 @@ module Langchain::Vectorsearch
|
|
36
36
|
status_code, response = @client.database.load_db(db_name, db_path)
|
37
37
|
|
38
38
|
if status_code != 200
|
39
|
-
if status_code == 500 && response["message"].include?("already loaded")
|
39
|
+
if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
|
40
|
+
# When db is already loaded, Epsilla may return HTTP 409 Conflict.
|
41
|
+
# This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
|
42
|
+
# Old behavior (HTTP 500) is preserved for backwards compatibility.
|
43
|
+
# It does not prevent us from using the db.
|
40
44
|
Langchain.logger.info("Database already loaded")
|
41
45
|
else
|
42
46
|
raise "Failed to load database: #{response}"
|
@@ -64,13 +64,13 @@ module Langchain::Vectorsearch
|
|
64
64
|
index.upsert(vectors: vectors, namespace: namespace)
|
65
65
|
end
|
66
66
|
|
67
|
-
def add_data(paths:, namespace: "")
|
67
|
+
def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
|
68
68
|
raise ArgumentError, "Paths must be provided" if Array(paths).empty?
|
69
69
|
|
70
70
|
texts = Array(paths)
|
71
71
|
.flatten
|
72
72
|
.map do |path|
|
73
|
-
data = Langchain::Loader.new(path)&.load&.chunks
|
73
|
+
data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
|
74
74
|
data.map { |chunk| chunk.text }
|
75
75
|
end
|
76
76
|
|
data/lib/langchain/version.rb
CHANGED
data/lib/langchain.rb
CHANGED
@@ -24,6 +24,7 @@ loader.inflector.inflect(
|
|
24
24
|
"sql_query_agent" => "SQLQueryAgent"
|
25
25
|
)
|
26
26
|
loader.collapse("#{__dir__}/langchain/llm/response")
|
27
|
+
loader.collapse("#{__dir__}/langchain/assistants")
|
27
28
|
loader.setup
|
28
29
|
|
29
30
|
# Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
|
@@ -82,7 +83,7 @@ module Langchain
|
|
82
83
|
attr_reader :root
|
83
84
|
end
|
84
85
|
|
85
|
-
self.logger ||= ::Logger.new($stdout, level: :
|
86
|
+
self.logger ||= ::Logger.new($stdout, level: :debug)
|
86
87
|
|
87
88
|
@root = Pathname.new(__dir__)
|
88
89
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: langchainrb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrei Bondarev
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-01-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: baran
|
@@ -534,14 +534,14 @@ dependencies:
|
|
534
534
|
requirements:
|
535
535
|
- - "~>"
|
536
536
|
- !ruby/object:Gem::Version
|
537
|
-
version: 6.
|
537
|
+
version: 6.3.0
|
538
538
|
type: :development
|
539
539
|
prerelease: false
|
540
540
|
version_requirements: !ruby/object:Gem::Requirement
|
541
541
|
requirements:
|
542
542
|
- - "~>"
|
543
543
|
- !ruby/object:Gem::Version
|
544
|
-
version: 6.
|
544
|
+
version: 6.3.0
|
545
545
|
- !ruby/object:Gem::Dependency
|
546
546
|
name: safe_ruby
|
547
547
|
requirement: !ruby/object:Gem::Requirement
|
@@ -616,8 +616,12 @@ files:
|
|
616
616
|
- lib/langchain/agent/sql_query_agent.rb
|
617
617
|
- lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.yaml
|
618
618
|
- lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
|
619
|
+
- lib/langchain/assistants/assistant.rb
|
620
|
+
- lib/langchain/assistants/message.rb
|
621
|
+
- lib/langchain/assistants/thread.rb
|
619
622
|
- lib/langchain/chunk.rb
|
620
623
|
- lib/langchain/chunker/base.rb
|
624
|
+
- lib/langchain/chunker/markdown.rb
|
621
625
|
- lib/langchain/chunker/prompts/semantic_prompt_template.yml
|
622
626
|
- lib/langchain/chunker/recursive_text.rb
|
623
627
|
- lib/langchain/chunker/semantic.rb
|
@@ -677,6 +681,7 @@ files:
|
|
677
681
|
- lib/langchain/processors/html.rb
|
678
682
|
- lib/langchain/processors/json.rb
|
679
683
|
- lib/langchain/processors/jsonl.rb
|
684
|
+
- lib/langchain/processors/markdown.rb
|
680
685
|
- lib/langchain/processors/pdf.rb
|
681
686
|
- lib/langchain/processors/text.rb
|
682
687
|
- lib/langchain/processors/xlsx.rb
|
@@ -736,7 +741,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
736
741
|
- !ruby/object:Gem::Version
|
737
742
|
version: '0'
|
738
743
|
requirements: []
|
739
|
-
rubygems_version: 3.
|
744
|
+
rubygems_version: 3.4.1
|
740
745
|
signing_key:
|
741
746
|
specification_version: 4
|
742
747
|
summary: Build LLM-backed Ruby applications with Ruby's LangChain
|