langchainrb 0.13.5 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +14 -0
  3. data/README.md +2 -17
  4. data/lib/langchain/assistants/assistant.rb +207 -92
  5. data/lib/langchain/assistants/messages/ollama_message.rb +74 -0
  6. data/lib/langchain/assistants/thread.rb +8 -1
  7. data/lib/langchain/contextual_logger.rb +2 -2
  8. data/lib/langchain/llm/ai21.rb +0 -4
  9. data/lib/langchain/llm/anthropic.rb +15 -6
  10. data/lib/langchain/llm/azure.rb +3 -3
  11. data/lib/langchain/llm/base.rb +1 -0
  12. data/lib/langchain/llm/cohere.rb +0 -2
  13. data/lib/langchain/llm/google_gemini.rb +1 -1
  14. data/lib/langchain/llm/google_palm.rb +1 -4
  15. data/lib/langchain/llm/ollama.rb +24 -18
  16. data/lib/langchain/llm/openai.rb +1 -1
  17. data/lib/langchain/llm/response/google_gemini_response.rb +1 -1
  18. data/lib/langchain/llm/response/ollama_response.rb +5 -1
  19. data/lib/langchain/llm/unified_parameters.rb +2 -2
  20. data/lib/langchain/tool/calculator.rb +38 -0
  21. data/lib/langchain/tool/{database/database.rb → database.rb} +24 -12
  22. data/lib/langchain/tool/file_system.rb +44 -0
  23. data/lib/langchain/tool/{google_search/google_search.rb → google_search.rb} +17 -23
  24. data/lib/langchain/tool/{news_retriever/news_retriever.rb → news_retriever.rb} +41 -14
  25. data/lib/langchain/tool/ruby_code_interpreter.rb +41 -0
  26. data/lib/langchain/tool/{tavily/tavily.rb → tavily.rb} +24 -10
  27. data/lib/langchain/tool/vectorsearch.rb +40 -0
  28. data/lib/langchain/tool/{weather/weather.rb → weather.rb} +21 -17
  29. data/lib/langchain/tool/{wikipedia/wikipedia.rb → wikipedia.rb} +17 -13
  30. data/lib/langchain/tool_definition.rb +212 -0
  31. data/lib/langchain/utils/hash_transformer.rb +9 -17
  32. data/lib/langchain/vectorsearch/chroma.rb +2 -2
  33. data/lib/langchain/vectorsearch/elasticsearch.rb +2 -2
  34. data/lib/langchain/vectorsearch/epsilla.rb +3 -3
  35. data/lib/langchain/vectorsearch/milvus.rb +3 -3
  36. data/lib/langchain/vectorsearch/pgvector.rb +2 -2
  37. data/lib/langchain/vectorsearch/pinecone.rb +2 -2
  38. data/lib/langchain/vectorsearch/qdrant.rb +2 -2
  39. data/lib/langchain/vectorsearch/weaviate.rb +4 -4
  40. data/lib/langchain/version.rb +1 -1
  41. metadata +16 -45
  42. data/lib/langchain/tool/base.rb +0 -107
  43. data/lib/langchain/tool/calculator/calculator.json +0 -19
  44. data/lib/langchain/tool/calculator/calculator.rb +0 -34
  45. data/lib/langchain/tool/database/database.json +0 -46
  46. data/lib/langchain/tool/file_system/file_system.json +0 -57
  47. data/lib/langchain/tool/file_system/file_system.rb +0 -32
  48. data/lib/langchain/tool/google_search/google_search.json +0 -19
  49. data/lib/langchain/tool/news_retriever/news_retriever.json +0 -122
  50. data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.json +0 -19
  51. data/lib/langchain/tool/ruby_code_interpreter/ruby_code_interpreter.rb +0 -37
  52. data/lib/langchain/tool/tavily/tavily.json +0 -54
  53. data/lib/langchain/tool/vectorsearch/vectorsearch.json +0 -24
  54. data/lib/langchain/tool/vectorsearch/vectorsearch.rb +0 -36
  55. data/lib/langchain/tool/weather/weather.json +0 -19
  56. data/lib/langchain/tool/wikipedia/wikipedia.json +0 -19
  57. data/lib/langchain/utils/token_length/ai21_validator.rb +0 -41
  58. data/lib/langchain/utils/token_length/base_validator.rb +0 -42
  59. data/lib/langchain/utils/token_length/cohere_validator.rb +0 -49
  60. data/lib/langchain/utils/token_length/google_palm_validator.rb +0 -57
  61. data/lib/langchain/utils/token_length/openai_validator.rb +0 -138
  62. data/lib/langchain/utils/token_length/token_limit_exceeded.rb +0 -17
@@ -1,41 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Langchain
4
- module Utils
5
- module TokenLength
6
- #
7
- # This class is meant to validate the length of the text passed in to AI21's API.
8
- # It is used to validate the token length before the API call is made
9
- #
10
-
11
- class AI21Validator < BaseValidator
12
- TOKEN_LIMITS = {
13
- "j2-ultra" => 8192,
14
- "j2-mid" => 8192,
15
- "j2-light" => 8192
16
- }.freeze
17
-
18
- #
19
- # Calculate token length for a given text and model name
20
- #
21
- # @param text [String] The text to calculate the token length for
22
- # @param model_name [String] The model name to validate against
23
- # @return [Integer] The token length of the text
24
- #
25
- def self.token_length(text, model_name, options = {})
26
- res = options[:llm].tokenize(text)
27
- res.dig(:tokens).length
28
- end
29
-
30
- def self.token_limit(model_name)
31
- TOKEN_LIMITS[model_name]
32
- end
33
- singleton_class.alias_method :completion_token_limit, :token_limit
34
-
35
- def self.token_length_from_messages(messages, model_name, options)
36
- messages.sum { |message| token_length(message.to_json, model_name, options) }
37
- end
38
- end
39
- end
40
- end
41
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Langchain
4
- module Utils
5
- module TokenLength
6
- #
7
- # Calculate the `max_tokens:` parameter to be set by calculating the context length of the text minus the prompt length
8
- #
9
- # @param content [String | Array<String>] The text or array of texts to validate
10
- # @param model_name [String] The model name to validate against
11
- # @return [Integer] Whether the text is valid or not
12
- # @raise [TokenLimitExceeded] If the text is too long
13
- #
14
- class BaseValidator
15
- def self.validate_max_tokens!(content, model_name, options = {})
16
- text_token_length = if content.is_a?(Array)
17
- token_length_from_messages(content, model_name, options)
18
- else
19
- token_length(content, model_name, options)
20
- end
21
-
22
- leftover_tokens = token_limit(model_name) - text_token_length
23
-
24
- # Some models have a separate token limit for completions (e.g. GPT-4 Turbo)
25
- # We want the lower of the two limits
26
- max_tokens = [leftover_tokens, completion_token_limit(model_name)].min
27
-
28
- # Raise an error even if whole prompt is equal to the model's token limit (leftover_tokens == 0)
29
- if max_tokens < 0
30
- raise limit_exceeded_exception(token_limit(model_name), text_token_length)
31
- end
32
-
33
- max_tokens
34
- end
35
-
36
- def self.limit_exceeded_exception(limit, length)
37
- TokenLimitExceeded.new("This model's maximum context length is #{limit} tokens, but the given text is #{length} tokens long.", length - limit)
38
- end
39
- end
40
- end
41
- end
42
- end
@@ -1,49 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Langchain
4
- module Utils
5
- module TokenLength
6
- #
7
- # This class is meant to validate the length of the text passed in to Cohere's API.
8
- # It is used to validate the token length before the API call is made
9
- #
10
-
11
- class CohereValidator < BaseValidator
12
- TOKEN_LIMITS = {
13
- # Source:
14
- # https://docs.cohere.com/docs/models
15
- "command-light" => 4096,
16
- "command" => 4096,
17
- "base-light" => 2048,
18
- "base" => 2048,
19
- "embed-english-light-v2.0" => 512,
20
- "embed-english-v2.0" => 512,
21
- "embed-multilingual-v2.0" => 256,
22
- "summarize-medium" => 2048,
23
- "summarize-xlarge" => 2048
24
- }.freeze
25
-
26
- #
27
- # Calculate token length for a given text and model name
28
- #
29
- # @param text [String] The text to calculate the token length for
30
- # @param model_name [String] The model name to validate against
31
- # @return [Integer] The token length of the text
32
- #
33
- def self.token_length(text, model_name, options = {})
34
- res = options[:llm].tokenize(text: text)
35
- res["tokens"].length
36
- end
37
-
38
- def self.token_limit(model_name)
39
- TOKEN_LIMITS[model_name]
40
- end
41
- singleton_class.alias_method :completion_token_limit, :token_limit
42
-
43
- def self.token_length_from_messages(messages, model_name, options)
44
- messages.sum { |message| token_length(message.to_json, model_name, options) }
45
- end
46
- end
47
- end
48
- end
49
- end
@@ -1,57 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Langchain
4
- module Utils
5
- module TokenLength
6
- #
7
- # This class is meant to validate the length of the text passed in to Google Palm's API.
8
- # It is used to validate the token length before the API call is made
9
- #
10
- class GooglePalmValidator < BaseValidator
11
- TOKEN_LIMITS = {
12
- # Source:
13
- # This data can be pulled when `list_models()` method is called: https://github.com/andreibondarev/google_palm_api#usage
14
-
15
- # chat-bison-001 is the only model that currently supports countMessageTokens functions
16
- "chat-bison-001" => {
17
- "input_token_limit" => 4000, # 4096 is the limit but the countMessageTokens does not return anything higher than 4000
18
- "output_token_limit" => 1024
19
- }
20
- # "text-bison-001" => {
21
- # "input_token_limit" => 8196,
22
- # "output_token_limit" => 1024
23
- # },
24
- # "embedding-gecko-001" => {
25
- # "input_token_limit" => 1024
26
- # }
27
- }.freeze
28
-
29
- #
30
- # Calculate token length for a given text and model name
31
- #
32
- # @param text [String] The text to calculate the token length for
33
- # @param model_name [String] The model name to validate against
34
- # @param options [Hash] the options to create a message with
35
- # @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
36
- # @return [Integer] The token length of the text
37
- #
38
- def self.token_length(text, model_name = "chat-bison-001", options = {})
39
- response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
40
-
41
- raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
42
-
43
- response.dig("tokenCount")
44
- end
45
-
46
- def self.token_length_from_messages(messages, model_name, options = {})
47
- messages.sum { |message| token_length(message.to_json, model_name, options) }
48
- end
49
-
50
- def self.token_limit(model_name)
51
- TOKEN_LIMITS.dig(model_name, "input_token_limit")
52
- end
53
- singleton_class.alias_method :completion_token_limit, :token_limit
54
- end
55
- end
56
- end
57
- end
@@ -1,138 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "tiktoken_ruby"
4
-
5
- module Langchain
6
- module Utils
7
- module TokenLength
8
- #
9
- # This class is meant to validate the length of the text passed in to OpenAI's API.
10
- # It is used to validate the token length before the API call is made
11
- #
12
- class OpenAIValidator < BaseValidator
13
- COMPLETION_TOKEN_LIMITS = {
14
- # GPT-4 Turbo has a separate token limit for completion
15
- # Source:
16
- # https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
17
- "gpt-4-1106-preview" => 4096,
18
- "gpt-4-vision-preview" => 4096,
19
- "gpt-3.5-turbo-1106" => 4096
20
- }
21
-
22
- # NOTE: The gpt-4-turbo-preview is an alias that will always point to the latest GPT 4 Turbo preview
23
- # the future previews may have a different token limit!
24
- TOKEN_LIMITS = {
25
- # Source:
26
- # https://platform.openai.com/docs/api-reference/embeddings
27
- # https://platform.openai.com/docs/models/gpt-4
28
- "text-embedding-3-large" => 8191,
29
- "text-embedding-3-small" => 8191,
30
- "text-embedding-ada-002" => 8191,
31
- "gpt-3.5-turbo" => 16385,
32
- "gpt-3.5-turbo-0301" => 4096,
33
- "gpt-3.5-turbo-0613" => 4096,
34
- "gpt-3.5-turbo-1106" => 16385,
35
- "gpt-3.5-turbo-0125" => 16385,
36
- "gpt-3.5-turbo-16k" => 16384,
37
- "gpt-3.5-turbo-16k-0613" => 16384,
38
- "text-davinci-003" => 4097,
39
- "text-davinci-002" => 4097,
40
- "code-davinci-002" => 8001,
41
- "gpt-4" => 8192,
42
- "gpt-4-0314" => 8192,
43
- "gpt-4-0613" => 8192,
44
- "gpt-4-32k" => 32768,
45
- "gpt-4-32k-0314" => 32768,
46
- "gpt-4-32k-0613" => 32768,
47
- "gpt-4-1106-preview" => 128000,
48
- "gpt-4-turbo" => 128000,
49
- "gpt-4-turbo-2024-04-09" => 128000,
50
- "gpt-4-turbo-preview" => 128000,
51
- "gpt-4-0125-preview" => 128000,
52
- "gpt-4-vision-preview" => 128000,
53
- "gpt-4o" => 128000,
54
- "gpt-4o-2024-05-13" => 128000,
55
- "text-curie-001" => 2049,
56
- "text-babbage-001" => 2049,
57
- "text-ada-001" => 2049,
58
- "davinci" => 2049,
59
- "curie" => 2049,
60
- "babbage" => 2049,
61
- "ada" => 2049
62
- }.freeze
63
-
64
- #
65
- # Calculate token length for a given text and model name
66
- #
67
- # @param text [String] The text to calculate the token length for
68
- # @param model_name [String] The model name to validate against
69
- # @return [Integer] The token length of the text
70
- #
71
- def self.token_length(text, model_name, options = {})
72
- # tiktoken-ruby doesn't support text-embedding-3-large or text-embedding-3-small yet
73
- if ["text-embedding-3-large", "text-embedding-3-small"].include?(model_name)
74
- model_name = "text-embedding-ada-002"
75
- end
76
-
77
- encoder = Tiktoken.encoding_for_model(model_name)
78
- encoder.encode(text).length
79
- end
80
-
81
- def self.token_limit(model_name)
82
- TOKEN_LIMITS[model_name]
83
- end
84
-
85
- def self.completion_token_limit(model_name)
86
- COMPLETION_TOKEN_LIMITS[model_name] || token_limit(model_name)
87
- end
88
-
89
- # If :max_tokens is passed in, take the lower of it and the calculated max_tokens
90
- def self.validate_max_tokens!(content, model_name, options = {})
91
- max_tokens = super(content, model_name, options)
92
- [options[:max_tokens], max_tokens].reject(&:nil?).min
93
- end
94
-
95
- # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
96
- # Return the number of tokens used by a list of messages
97
- #
98
- # @param messages [Array<Hash>] The messages to calculate the token length for
99
- # @param model [String] The model name to validate against
100
- # @return [Integer] The token length of the messages
101
- #
102
- def self.token_length_from_messages(messages, model_name, options = {})
103
- encoding = Tiktoken.encoding_for_model(model_name)
104
-
105
- if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
106
- tokens_per_message = 3
107
- tokens_per_name = 1
108
- elsif model_name == "gpt-3.5-turbo-0301"
109
- tokens_per_message = 4 # every message follows {role/name}\n{content}\n
110
- tokens_per_name = -1 # if there's a name, the role is omitted
111
- elsif model_name.include?("gpt-3.5-turbo")
112
- # puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
113
- return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
114
- elsif model_name.include?("gpt-4")
115
- # puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
116
- return token_length_from_messages(messages, "gpt-4-0613", options)
117
- else
118
- raise NotImplementedError.new(
119
- "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
120
- )
121
- end
122
-
123
- num_tokens = 0
124
- messages.each do |message|
125
- num_tokens += tokens_per_message
126
- message.each do |key, value|
127
- num_tokens += encoding.encode(value).length
128
- num_tokens += tokens_per_name if ["name", :name].include?(key)
129
- end
130
- end
131
-
132
- num_tokens += 3 # every reply is primed with assistant
133
- num_tokens
134
- end
135
- end
136
- end
137
- end
138
- end
@@ -1,17 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Langchain
4
- module Utils
5
- module TokenLength
6
- class TokenLimitExceeded < StandardError
7
- attr_reader :token_overflow
8
-
9
- def initialize(message = "", token_overflow = 0)
10
- super(message)
11
-
12
- @token_overflow = token_overflow
13
- end
14
- end
15
- end
16
- end
17
- end