langchainrb 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/README.md +53 -25
  4. data/lib/langchain/assistants/assistant.rb +199 -0
  5. data/lib/langchain/assistants/message.rb +58 -0
  6. data/lib/langchain/assistants/thread.rb +34 -0
  7. data/lib/langchain/chunker/markdown.rb +39 -0
  8. data/lib/langchain/conversation/memory.rb +1 -6
  9. data/lib/langchain/conversation.rb +7 -18
  10. data/lib/langchain/data.rb +4 -3
  11. data/lib/langchain/llm/ai21.rb +1 -1
  12. data/lib/langchain/llm/azure.rb +10 -97
  13. data/lib/langchain/llm/base.rb +1 -0
  14. data/lib/langchain/llm/cohere.rb +4 -6
  15. data/lib/langchain/llm/google_palm.rb +2 -0
  16. data/lib/langchain/llm/google_vertex_ai.rb +12 -10
  17. data/lib/langchain/llm/openai.rb +104 -160
  18. data/lib/langchain/llm/replicate.rb +0 -6
  19. data/lib/langchain/llm/response/anthropic_response.rb +4 -0
  20. data/lib/langchain/llm/response/google_palm_response.rb +4 -0
  21. data/lib/langchain/llm/response/ollama_response.rb +5 -1
  22. data/lib/langchain/llm/response/openai_response.rb +8 -0
  23. data/lib/langchain/loader.rb +3 -2
  24. data/lib/langchain/processors/markdown.rb +17 -0
  25. data/lib/langchain/tool/base.rb +24 -0
  26. data/lib/langchain/tool/google_search.rb +1 -4
  27. data/lib/langchain/utils/token_length/ai21_validator.rb +6 -2
  28. data/lib/langchain/utils/token_length/base_validator.rb +1 -1
  29. data/lib/langchain/utils/token_length/cohere_validator.rb +6 -2
  30. data/lib/langchain/utils/token_length/google_palm_validator.rb +5 -1
  31. data/lib/langchain/utils/token_length/openai_validator.rb +41 -0
  32. data/lib/langchain/vectorsearch/base.rb +2 -2
  33. data/lib/langchain/vectorsearch/epsilla.rb +5 -1
  34. data/lib/langchain/vectorsearch/pinecone.rb +2 -2
  35. data/lib/langchain/version.rb +1 -1
  36. data/lib/langchain.rb +2 -1
  37. metadata +10 -5
@@ -91,6 +91,30 @@ module Langchain::Tool
91
91
  new.execute(input: input)
92
92
  end
93
93
 
94
+ # Returns the tool as an OpenAI tool
95
+ #
96
+ # @return [Hash] tool as an OpenAI tool
97
+ def to_openai_tool
98
+ # TODO: This is hardcoded to def execute(input:) found in each tool, needs to be dynamic.
99
+ {
100
+ type: "function",
101
+ function: {
102
+ name: name,
103
+ description: description,
104
+ parameters: {
105
+ type: "object",
106
+ properties: {
107
+ input: {
108
+ type: "string",
109
+ description: "Input to the tool"
110
+ }
111
+ },
112
+ required: ["input"]
113
+ }
114
+ }
115
+ }
116
+ end
117
+
94
118
  #
95
119
  # Executes the tool and returns the answer
96
120
  #
@@ -17,10 +17,7 @@ module Langchain::Tool
17
17
  description <<~DESC
18
18
  A wrapper around SerpApi's Google Search API.
19
19
 
20
- Useful for when you need to answer questions about current events.
21
- Always one of the first options when you need to find information on internet.
22
-
23
- Input should be a search query.
20
+ Useful for when you need to answer questions about current events. Always one of the first options when you need to find information on internet. Input should be a search query.
24
21
  DESC
25
22
 
26
23
  attr_reader :api_key
@@ -22,8 +22,8 @@ module Langchain
22
22
  # @param model_name [String] The model name to validate against
23
23
  # @return [Integer] The token length of the text
24
24
  #
25
- def self.token_length(text, model_name, client)
26
- res = client.tokenize(text)
25
+ def self.token_length(text, model_name, options = {})
26
+ res = options[:llm].tokenize(text)
27
27
  res.dig(:tokens).length
28
28
  end
29
29
 
@@ -31,6 +31,10 @@ module Langchain
31
31
  TOKEN_LIMITS[model_name]
32
32
  end
33
33
  singleton_class.alias_method :completion_token_limit, :token_limit
34
+
35
+ def self.token_length_from_messages(messages, model_name, options)
36
+ messages.sum { |message| token_length(message.to_json, model_name, options) }
37
+ end
34
38
  end
35
39
  end
36
40
  end
@@ -14,7 +14,7 @@ module Langchain
14
14
  class BaseValidator
15
15
  def self.validate_max_tokens!(content, model_name, options = {})
16
16
  text_token_length = if content.is_a?(Array)
17
- content.sum { |item| token_length(item.to_json, model_name, options) }
17
+ token_length_from_messages(content, model_name, options)
18
18
  else
19
19
  token_length(content, model_name, options)
20
20
  end
@@ -30,8 +30,8 @@ module Langchain
30
30
  # @param model_name [String] The model name to validate against
31
31
  # @return [Integer] The token length of the text
32
32
  #
33
- def self.token_length(text, model_name, client)
34
- res = client.tokenize(text: text)
33
+ def self.token_length(text, model_name, options = {})
34
+ res = options[:llm].tokenize(text: text)
35
35
  res["tokens"].length
36
36
  end
37
37
 
@@ -39,6 +39,10 @@ module Langchain
39
39
  TOKEN_LIMITS[model_name]
40
40
  end
41
41
  singleton_class.alias_method :completion_token_limit, :token_limit
42
+
43
+ def self.token_length_from_messages(messages, model_name, options)
44
+ messages.sum { |message| token_length(message.to_json, model_name, options) }
45
+ end
42
46
  end
43
47
  end
44
48
  end
@@ -35,7 +35,7 @@ module Langchain
35
35
  # @option options [Langchain::LLM:GooglePalm] :llm The Langchain::LLM:GooglePalm instance
36
36
  # @return [Integer] The token length of the text
37
37
  #
38
- def self.token_length(text, model_name = "chat-bison-001", options)
38
+ def self.token_length(text, model_name = "chat-bison-001", options = {})
39
39
  response = options[:llm].client.count_message_tokens(model: model_name, prompt: text)
40
40
 
41
41
  raise Langchain::LLM::ApiError.new(response["error"]["message"]) unless response["error"].nil?
@@ -43,6 +43,10 @@ module Langchain
43
43
  response.dig("tokenCount")
44
44
  end
45
45
 
46
+ def self.token_length_from_messages(messages, model_name, options = {})
47
+ messages.sum { |message| token_length(message.to_json, model_name, options) }
48
+ end
49
+
46
50
  def self.token_limit(model_name)
47
51
  TOKEN_LIMITS.dig(model_name, "input_token_limit")
48
52
  end
@@ -75,6 +75,47 @@ module Langchain
75
75
  max_tokens = super(content, model_name, options)
76
76
  [options[:max_tokens], max_tokens].reject(&:nil?).min
77
77
  end
78
+
79
+ # Copied from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
80
+ # Return the number of tokens used by a list of messages
81
+ #
82
+ # @param messages [Array<Hash>] The messages to calculate the token length for
83
+ # @param model [String] The model name to validate against
84
+ # @return [Integer] The token length of the messages
85
+ #
86
+ def self.token_length_from_messages(messages, model_name, options = {})
87
+ encoding = Tiktoken.encoding_for_model(model_name)
88
+
89
+ if ["gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k-0613", "gpt-4-0314", "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613"].include?(model_name)
90
+ tokens_per_message = 3
91
+ tokens_per_name = 1
92
+ elsif model_name == "gpt-3.5-turbo-0301"
93
+ tokens_per_message = 4 # every message follows {role/name}\n{content}\n
94
+ tokens_per_name = -1 # if there's a name, the role is omitted
95
+ elsif model_name.include?("gpt-3.5-turbo")
96
+ # puts "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613."
97
+ return token_length_from_messages(messages, "gpt-3.5-turbo-0613", options)
98
+ elsif model_name.include?("gpt-4")
99
+ # puts "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613."
100
+ return token_length_from_messages(messages, "gpt-4-0613", options)
101
+ else
102
+ raise NotImplementedError.new(
103
+ "token_length_from_messages() is not implemented for model #{model_name}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."
104
+ )
105
+ end
106
+
107
+ num_tokens = 0
108
+ messages.each do |message|
109
+ num_tokens += tokens_per_message
110
+ message.each do |key, value|
111
+ num_tokens += encoding.encode(value).length
112
+ num_tokens += tokens_per_name if ["name", :name].include?(key)
113
+ end
114
+ end
115
+
116
+ num_tokens += 3 # every reply is primed with assistant
117
+ num_tokens
118
+ end
78
119
  end
79
120
  end
80
121
  end
@@ -175,13 +175,13 @@ module Langchain::Vectorsearch
175
175
  prompt_template.format(question: question, context: context)
176
176
  end
177
177
 
178
- def add_data(paths:)
178
+ def add_data(paths:, options: {}, chunker: Langchain::Chunker::Text)
179
179
  raise ArgumentError, "Paths must be provided" if Array(paths).empty?
180
180
 
181
181
  texts = Array(paths)
182
182
  .flatten
183
183
  .map do |path|
184
- data = Langchain::Loader.new(path)&.load&.chunks
184
+ data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
185
185
  data.map { |chunk| chunk.text }
186
186
  end
187
187
 
@@ -36,7 +36,11 @@ module Langchain::Vectorsearch
36
36
  status_code, response = @client.database.load_db(db_name, db_path)
37
37
 
38
38
  if status_code != 200
39
- if status_code == 500 && response["message"].include?("already loaded")
39
+ if status_code == 409 || (status_code == 500 && response["message"].include?("already loaded"))
40
+ # When db is already loaded, Epsilla may return HTTP 409 Conflict.
41
+ # This behavior is changed in https://github.com/epsilla-cloud/vectordb/pull/95
42
+ # Old behavior (HTTP 500) is preserved for backwards compatibility.
43
+ # It does not prevent us from using the db.
40
44
  Langchain.logger.info("Database already loaded")
41
45
  else
42
46
  raise "Failed to load database: #{response}"
@@ -64,13 +64,13 @@ module Langchain::Vectorsearch
64
64
  index.upsert(vectors: vectors, namespace: namespace)
65
65
  end
66
66
 
67
- def add_data(paths:, namespace: "")
67
+ def add_data(paths:, namespace: "", options: {}, chunker: Langchain::Chunker::Text)
68
68
  raise ArgumentError, "Paths must be provided" if Array(paths).empty?
69
69
 
70
70
  texts = Array(paths)
71
71
  .flatten
72
72
  .map do |path|
73
- data = Langchain::Loader.new(path)&.load&.chunks
73
+ data = Langchain::Loader.new(path, options, chunker: chunker)&.load&.chunks
74
74
  data.map { |chunk| chunk.text }
75
75
  end
76
76
 
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Langchain
4
- VERSION = "0.8.1"
4
+ VERSION = "0.9.0"
5
5
  end
data/lib/langchain.rb CHANGED
@@ -24,6 +24,7 @@ loader.inflector.inflect(
24
24
  "sql_query_agent" => "SQLQueryAgent"
25
25
  )
26
26
  loader.collapse("#{__dir__}/langchain/llm/response")
27
+ loader.collapse("#{__dir__}/langchain/assistants")
27
28
  loader.setup
28
29
 
29
30
  # Langchain.rb a is library for building LLM-backed Ruby applications. It is an abstraction layer that sits on top of the emerging AI-related tools that makes it easy for developers to consume and string those services together.
@@ -82,7 +83,7 @@ module Langchain
82
83
  attr_reader :root
83
84
  end
84
85
 
85
- self.logger ||= ::Logger.new($stdout, level: :warn)
86
+ self.logger ||= ::Logger.new($stdout, level: :debug)
86
87
 
87
88
  @root = Pathname.new(__dir__)
88
89
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: langchainrb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.1
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrei Bondarev
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-12-07 00:00:00.000000000 Z
11
+ date: 2024-01-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: baran
@@ -534,14 +534,14 @@ dependencies:
534
534
  requirements:
535
535
  - - "~>"
536
536
  - !ruby/object:Gem::Version
537
- version: 6.1.0
537
+ version: 6.3.0
538
538
  type: :development
539
539
  prerelease: false
540
540
  version_requirements: !ruby/object:Gem::Requirement
541
541
  requirements:
542
542
  - - "~>"
543
543
  - !ruby/object:Gem::Version
544
- version: 6.1.0
544
+ version: 6.3.0
545
545
  - !ruby/object:Gem::Dependency
546
546
  name: safe_ruby
547
547
  requirement: !ruby/object:Gem::Requirement
@@ -616,8 +616,12 @@ files:
616
616
  - lib/langchain/agent/sql_query_agent.rb
617
617
  - lib/langchain/agent/sql_query_agent/sql_query_agent_answer_prompt.yaml
618
618
  - lib/langchain/agent/sql_query_agent/sql_query_agent_sql_prompt.yaml
619
+ - lib/langchain/assistants/assistant.rb
620
+ - lib/langchain/assistants/message.rb
621
+ - lib/langchain/assistants/thread.rb
619
622
  - lib/langchain/chunk.rb
620
623
  - lib/langchain/chunker/base.rb
624
+ - lib/langchain/chunker/markdown.rb
621
625
  - lib/langchain/chunker/prompts/semantic_prompt_template.yml
622
626
  - lib/langchain/chunker/recursive_text.rb
623
627
  - lib/langchain/chunker/semantic.rb
@@ -677,6 +681,7 @@ files:
677
681
  - lib/langchain/processors/html.rb
678
682
  - lib/langchain/processors/json.rb
679
683
  - lib/langchain/processors/jsonl.rb
684
+ - lib/langchain/processors/markdown.rb
680
685
  - lib/langchain/processors/pdf.rb
681
686
  - lib/langchain/processors/text.rb
682
687
  - lib/langchain/processors/xlsx.rb
@@ -736,7 +741,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
736
741
  - !ruby/object:Gem::Version
737
742
  version: '0'
738
743
  requirements: []
739
- rubygems_version: 3.3.7
744
+ rubygems_version: 3.4.1
740
745
  signing_key:
741
746
  specification_version: 4
742
747
  summary: Build LLM-backed Ruby applications with Ruby's LangChain