langchainrb 0.13.5 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/README.md +2 -17
- data/lib/langchain/assistants/assistant.rb +207 -92
- data/lib/langchain/assistants/messages/ollama_message.rb +74 -0
- data/lib/langchain/assistants/thread.rb +8 -1
- data/lib/langchain/contextual_logger.rb +2 -2
- data/lib/langchain/llm/ai21.rb +0 -4
- data/lib/langchain/llm/anthropic.rb +15 -6
- data/lib/langchain/llm/azure.rb +3 -3
- data/lib/langchain/llm/base.rb +1 -0
- data/lib/langchain/llm/cohere.rb +0 -2
- data/lib/langchain/llm/google_gemini.rb +1 -1
- data/lib/langchain/llm/google_palm.rb +1 -4
- data/lib/langchain/llm/ollama.rb +24 -18
- data/lib/langchain/llm/openai.rb +1 -1
- data/lib/langchain/llm/response/google_gemini_response.rb +1 -1
- data/lib/langchain/llm/response/ollama_response.rb +5 -1
- data/lib/langchain/llm/unified_parameters.rb +2 -2
- data/lib/langchain/tool/calculator.rb +38 -0
- data/lib/langchain/tool/{database/database.rb → database.rb} +24 -12
- data/lib/langchain/tool/file_system.rb +44 -0
- data/lib/langchain/tool/{google_search/google_search.rb → google_search.rb} +17 -23
- data/lib/langchain/tool/{news_retriever/news_retriever.rb → news_retriever.rb} +41 -14
- data/lib/langchain/tool/ruby_code_interpreter.rb +41 -0
- data/lib/langchain/tool/{tavily/tavily.rb → tavily.rb} +24 -10
- data/lib/langchain/tool/vectorsearch.rb +40 -0
- data/lib/langchain/tool/{weather/weather.rb → weather.rb} +21 -17
- data/lib/langchain/tool/{wikipedia/wikipedia.rb → wikipedia.rb} +17 -13
- data/lib/langchain/tool_definition.rb +212 -0
- data/lib/langchain/utils/hash_transformer.rb +9 -17
- data/lib/langchain/vectorsearch/chroma.rb +2 -2
- data/lib/langchain/vectorsearch/elasticsearch.rb +2 -2
- data/lib/langchain/vectorsearch/epsilla.rb +3 -3
- data/lib/langchain/vectorsearch/milvus.rb +3 -3
- data/lib/langchain/vectorsearch/pgvector.rb +2 -2
- data/lib/langchain/vectorsearch/pinecone.rb +2 -2
- data/lib/langchain/vectorsearch/qdrant.rb +2 -2
- data/lib/langchain/vectorsearch/weaviate.rb +4 -4
- data/lib/langchain/version.rb +1 -1
- metadata +16 -45
- data/lib/langchain/tool/base.rb +0 -107
- data/lib/langchain/tool/calculator/calculator.json +0 -19
- data/lib/langchain/tool/calculator/calculator.rb +0 -34
- data/lib/langchain/tool/database/database.json +0 -46
- data/lib/langchain/tool/file_system/file_system.json +0 -57
- data/lib/langchain/tool/file_system/file_system.rb +0 -32
- data/lib/langchain/tool/google_search/google_search.json +0 -19
- data/lib/langchain/tool/news_retriever/news_retriever.json +0 -122
- data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.json +0 -19
- data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.rb +0 -37
- data/lib/langchain/tool/tavily/tavily.json +0 -54
- data/lib/langchain/tool/vectorsearch/vectorsearch.json +0 -24
- data/lib/langchain/tool/vectorsearch/vectorsearch.rb +0 -36
- data/lib/langchain/tool/weather/weather.json +0 -19
- data/lib/langchain/tool/wikipedia/wikipedia.json +0 -19
- data/lib/langchain/utils/token_length/ai21_validator.rb +0 -41
- data/lib/langchain/utils/token_length/base_validator.rb +0 -42
- data/lib/langchain/utils/token_length/cohere_validator.rb +0 -49
- data/lib/langchain/utils/token_length/google_palm_validator.rb +0 -57
- data/lib/langchain/utils/token_length/openai_validator.rb +0 -138
- data/lib/langchain/utils/token_length/token_limit_exceeded.rb +0 -17
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Langchain
|
4
|
-
module Utils
|
5
|
-
module TokenLength
|
6
|
-
#
|
7
|
-
# This class is meant to validate the length of the text passed in to AI21's API.
|
8
|
-
# It is used to validate the token length before the API call is made
|
9
|
-
#
|
10
|
-
|
11
|
-
class AI21Validator < BaseValidator
|
12
|
-
TOKEN_LIMITS = {
|
13
|
-
"j2-ultra" => 8192,
|
14
|
-
"j2-mid" => 8192,
|
15
|
-
"j2-light" => 8192
|
16
|
-
}.freeze
|
17
|
-
|
18
|
-
#
|
19
|
-
# Calculate token length for a given text and model name
|
20
|
-
#
|
21
|
-
# @param text [String] The text to calculate the token length for
|
22
|
-
# @param model_name [String] The model name to validate against
|
23
|
-
# @return [Integer] The token length of the text
|
24
|
-
#
|
25
|
-
def self.token_length(text, model_name, options = {})
|
26
|
-
res = options[:llm].tokenize(text)
|
27
|
-
res.dig(:tokens).length
|
28
|
-
end
|
29
|
-
|
30
|
-
def self.token_limit(model_name)
|
31
|
-
TOKEN_LIMITS[model_name]
|
32
|
-
end
|
33
|
-
singleton_class.alias_method :completion_token_limit, :token_limit
|
34
|
-
|
35
|
-
def self.token_length_from_messages(messages, model_name, options)
|
36
|
-
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Langchain
|
4
|
-
module Utils
|
5
|
-
module TokenLength
|
6
|
-
#
|
7
|
-
# Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
|
8
|
-
#
|
9
|
-
# @param content [String | Array<String>] The text or array of texts to validate
|
10
|
-
# @param model_name [String] The model name to validate against
|
11
|
-
# @return [Integer] Whether the text is valid or not
|
12
|
-
# @raise [TokenLimitExceeded] If the text is too long
|
13
|
-
#
|
14
|
-
class BaseValidator
|
15
|
-
def self.validate_max_tokens!(content, model_name, options = {})
|
16
|
-
text_token_length = if content.is_a?(Array)
|
17
|
-
token_length_from_messages(content, model_name, options)
|
18
|
-
else
|
19
|
-
token_length(content, model_name, options)
|
20
|
-
end
|
21
|
-
|
22
|
-
leftover_tokens = token_limit(model_name) - text_token_length
|
23
|
-
|
24
|
-
# Some models have a separate token limit for completions (e.g. GPT-4 Turbo)
|
25
|
-
# We want the lower of the two limits
|
26
|
-
max_tokens = [leftover_tokens, completion_token_limit(model_name)].min
|
27
|
-
|
28
|
-
# Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
|
29
|
-
if max_tokens < 0
|
30
|
-
raise limit_exceeded_exception(token_limit(model_name), text_token_length)
|
31
|
-
end
|
32
|
-
|
33
|
-
max_tokens
|
34
|
-
end
|
35
|
-
|
36
|
-
def self.limit_exceeded_exception(limit, length)
|
37
|
-
TokenLimitExceeded.new("This model's maximum context length is #{limit} tokens, but the given text is #{length} tokens long.", length - limit)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,49 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Langchain
|
4
|
-
module Utils
|
5
|
-
module TokenLength
|
6
|
-
#
|
7
|
-
# This class is meant to validate the length of the text passed in to Cohere's API.
|
8
|
-
# It is used to validate the token length before the API call is made
|
9
|
-
#
|
10
|
-
|
11
|
-
class CohereValidator < BaseValidator
|
12
|
-
TOKEN_LIMITS = {
|
13
|
-
# Source:
|
14
|
-
# https://docs.cohere.com/docs/models
|
15
|
-
"command-light" => 4096,
|
16
|
-
"command" => 4096,
|
17
|
-
"base-light" => 2048,
|
18
|
-
"base" => 2048,
|
19
|
-
"embed-english-light-v2.0" => 512,
|
20
|
-
"embed-english-v2.0" => 512,
|
21
|
-
"embed-multilingual-v2.0" => 256,
|
22
|
-
"summarize-medium" => 2048,
|
23
|
-
"summarize-xlarge" => 2048
|
24
|
-
}.freeze
|
25
|
-
|
26
|
-
#
|
27
|
-
# Calculate token length for a given text and model name
|
28
|
-
#
|
29
|
-
# @param text [String] The text to calculate the token length for
|
30
|
-
# @param model_name [String] The model name to validate against
|
31
|
-
# @return [Integer] The token length of the text
|
32
|
-
#
|
33
|
-
def self.token_length(text, model_name, options = {})
|
34
|
-
res = options[:llm].tokenize(text: text)
|
35
|
-
res["tokens"].length
|
36
|
-
end
|
37
|
-
|
38
|
-
def self.token_limit(model_name)
|
39
|
-
TOKEN_LIMITS[model_name]
|
40
|
-
end
|
41
|
-
singleton_class.alias_method :completion_token_limit, :token_limit
|
42
|
-
|
43
|
-
def self.token_length_from_messages(messages, model_name, options)
|
44
|
-
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Langchain
|
4
|
-
module Utils
|
5
|
-
module TokenLength
|
6
|
-
#
|
7
|
-
# This class is meant to validate the length of the text passed in to Google Palm's API.
|
8
|
-
# It is used to validate the token length before the API call is made
|
9
|
-
#
|
10
|
-
class GooglePalmValidator < BaseValidator
|
11
|
-
TOKEN_LIMITS = {
|
12
|
-
# Source:
|
13
|
-
# This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
|
14
|
-
|
15
|
-
# chat-bison-001 is the only model that currently supports countMessageTokens functions
|
16
|
-
"chat-bison-001" => {
|
17
|
-
"input_token_limit" => 4000, # 4096 is the limit but the countMessageTokens does not return anything higher than 4000
|
18
|
-
"output_token_limit" => 1024
|
19
|
-
}
|
20
|
-
# "text-bison-001" => {
|
21
|
-
# "input_token_limit" => 8196,
|
22
|
-
# "output_token_limit" => 1024
|
23
|
-
# },
|
24
|
-
# "embedding-gecko-001" => {
|
25
|
-
# "input_token_limit" => 1024
|
26
|
-
# }
|
27
|
-
}.freeze
|
28
|
-
|
29
|
-
#
|
30
|
-
# Calculate token length for a given text and model name
|
31
|
-
#
|
32
|
-
# @param text [String] The text to calculate the token length for
|
33
|
-
# @param model_name [String] The model name to validate against
|
34
|
-
# @param options [Hash] the options to create a message with
|
35
|
-
# @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
|
36
|
-
# @return [Integer] The token length of the text
|
37
|
-
#
|
38
|
-
def self.token_length(text, model_name = "chat-bison-001", options = {})
|
39
|
-
response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
|
40
|
-
|
41
|
-
raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
|
42
|
-
|
43
|
-
response.dig("tokenCount")
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.token_length_from_messages(messages, model_name, options = {})
|
47
|
-
messages.sum { |message| token_length(message.to_json, model_name, options) }
|
48
|
-
end
|
49
|
-
|
50
|
-
def self.token_limit(model_name)
|
51
|
-
TOKEN_LIMITS.dig(model_name, "input_token_limit")
|
52
|
-
end
|
53
|
-
singleton_class.alias_method :completion_token_limit, :token_limit
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,138 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "tiktoken_ruby"
|
4
|
-
|
5
|
-
module Langchain
|
6
|
-
module Utils
|
7
|
-
module TokenLength
|
8
|
-
#
|
9
|
-
# This class is meant to validate the length of the text passed in to OpenAI's API.
|
10
|
-
# It is used to validate the token length before the API call is made
|
11
|
-
#
|
12
|
-
class OpenAIValidator < BaseValidator
|
13
|
-
COMPLETION_TOKEN_LIMITS = {
|
14
|
-
# GPT-4 Turbo has a separate token limit for completion
|
15
|
-
# Source:
|
16
|
-
# https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
|
17
|
-
"gpt-4-1106-preview" => 4096,
|
18
|
-
"gpt-4-vision-preview" => 4096,
|
19
|
-
"gpt-3.5-turbo-1106" => 4096
|
20
|
-
}
|
21
|
-
|
22
|
-
# NOTE: The gpt-4-turbo-preview is an alias that will always point to the latest GPT 4 Turbo preview
|
23
|
-
# the future previews may have a different token limit!
|
24
|
-
TOKEN_LIMITS = {
|
25
|
-
# Source:
|
26
|
-
# https://platform.openai.com/docs/api-reference/embeddings
|
27
|
-
# https://platform.openai.com/docs/models/gpt-4
|
28
|
-
"text-embedding-3-large" => 8191,
|
29
|
-
"text-embedding-3-small" => 8191,
|
30
|
-
"text-embedding-ada-002" => 8191,
|
31
|
-
"gpt-3.5-turbo" => 16385,
|
32
|
-
"gpt-3.5-turbo-0301" => 4096,
|
33
|
-
"gpt-3.5-turbo-0613" => 4096,
|
34
|
-
"gpt-3.5-turbo-1106" => 16385,
|
35
|
-
"gpt-3.5-turbo-0125" => 16385,
|
36
|
-
"gpt-3.5-turbo-16k" => 16384,
|
37
|
-
"gpt-3.5-turbo-16k-0613" => 16384,
|
38
|
-
"text-davinci-003" => 4097,
|
39
|
-
"text-davinci-002" => 4097,
|
40
|
-
"code-davinci-002" => 8001,
|
41
|
-
"gpt-4" => 8192,
|
42
|
-
"gpt-4-0314" => 8192,
|
43
|
-
"gpt-4-0613" => 8192,
|
44
|
-
"gpt-4-32k" => 32768,
|
45
|
-
"gpt-4-32k-0314" => 32768,
|
46
|
-
"gpt-4-32k-0613" => 32768,
|
47
|
-
"gpt-4-1106-preview" => 128000,
|
48
|
-
"gpt-4-turbo" => 128000,
|
49
|
-
"gpt-4-turbo-2024-04-09" => 128000,
|
50
|
-
"gpt-4-turbo-preview" => 128000,
|
51
|
-
"gpt-4-0125-preview" => 128000,
|
52
|
-
"gpt-4-vision-preview" => 128000,
|
53
|
-
"gpt-4o" => 128000,
|
54
|
-
"gpt-4o-2024-05-13" => 128000,
|
55
|
-
"text-curie-001" => 2049,
|
56
|
-
"text-babbage-001" => 2049,
|
57
|
-
"text-ada-001" => 2049,
|
58
|
-
"davinci" => 2049,
|
59
|
-
"curie" => 2049,
|
60
|
-
"babbage" => 2049,
|
61
|
-
"ada" => 2049
|
62
|
-
}.freeze
|
63
|
-
|
64
|
-
#
|
65
|
-
# Calculate token length for a given text and model name
|
66
|
-
#
|
67
|
-
# @param text [String] The text to calculate the token length for
|
68
|
-
# @param model_name [String] The model name to validate against
|
69
|
-
# @return [Integer] The token length of the text
|
70
|
-
#
|
71
|
-
def self.token_length(text, model_name, options = {})
|
72
|
-
# tiktoken-ruby doesn't support text-embedding-3-large or text-embedding-3-small yet
|
73
|
-
if ["text-embedding-3-large", "text-embedding-3-small"].include?(model_name)
|
74
|
-
model_name = "text-embedding-ada-002"
|
75
|
-
end
|
76
|
-
|
77
|
-
encoder = Tiktoken.encoding_for_model(model_name)
|
78
|
-
encoder.encode(text).length
|
79
|
-
end
|
80
|
-
|
81
|
-
def self.token_limit(model_name)
|
82
|
-
TOKEN_LIMITS[model_name]
|
83
|
-
end
|
84
|
-
|
85
|
-
def self.completion_token_limit(model_name)
|
86
|
-
COMPLETION_TOKEN_LIMITS[model_name] || token_limit(model_name)
|
87
|
-
end
|
88
|
-
|
89
|
-
# If :max_tokens is passed in, take the lower of it and the calculated max_tokens
|
90
|
-
def self.validate_max_tokens!(content, model_name, options = {})
|
91
|
-
max_tokens = super(content, model_name, options)
|
92
|
-
[options[:max_tokens], max_tokens].reject(&:nil?).min
|
93
|
-
end
|
94
|
-
|
95
|
-
# Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
|
96
|
-
# Return the number of tokens used by a list of messages
|
97
|
-
#
|
98
|
-
# @param messages [Array<Hash>] The messages to calculate the token length for
|
99
|
-
# @param model [String] The model name to validate against
|
100
|
-
# @return [Integer] The token length of the messages
|
101
|
-
#
|
102
|
-
def self.token_length_from_messages(messages, model_name, options = {})
|
103
|
-
encoding = Tiktoken.encoding_for_model(model_name)
|
104
|
-
|
105
|
-
if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
|
106
|
-
tokens_per_message = 3
|
107
|
-
tokens_per_name = 1
|
108
|
-
elsif model_name == "gpt-3.5-turbo-0301"
|
109
|
-
tokens_per_message = 4 # every message follows {role/name}\n{content}\n
|
110
|
-
tokens_per_name = -1 # if there's a name, the role is omitted
|
111
|
-
elsif model_name.include?("gpt-3.5-turbo")
|
112
|
-
# puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
|
113
|
-
return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
|
114
|
-
elsif model_name.include?("gpt-4")
|
115
|
-
# puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
|
116
|
-
return token_length_from_messages(messages, "gpt-4-0613", options)
|
117
|
-
else
|
118
|
-
raise NotImplementedError.new(
|
119
|
-
"token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
|
120
|
-
)
|
121
|
-
end
|
122
|
-
|
123
|
-
num_tokens = 0
|
124
|
-
messages.each do |message|
|
125
|
-
num_tokens += tokens_per_message
|
126
|
-
message.each do |key, value|
|
127
|
-
num_tokens += encoding.encode(value).length
|
128
|
-
num_tokens += tokens_per_name if ["name", :name].include?(key)
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
num_tokens += 3 # every reply is primed with assistant
|
133
|
-
num_tokens
|
134
|
-
end
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
@@ -1,17 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Langchain
|
4
|
-
module Utils
|
5
|
-
module TokenLength
|
6
|
-
class TokenLimitExceeded < StandardError
|
7
|
-
attr_reader :token_overflow
|
8
|
-
|
9
|
-
def initialize(message = "", token_overflow = 0)
|
10
|
-
super(message)
|
11
|
-
|
12
|
-
@token_overflow = token_overflow
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|