langchainrb 0.3.10 → 0.3.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.env.example +1 -0
- data/CHANGELOG.md +5 -0
- data/Gemfile +0 -4
- data/Gemfile.lock +12 -13
- data/README.md +10 -3
- data/lib/dependency_helper.rb +8 -4
- data/lib/langchain.rb +3 -0
- data/lib/llm/base.rb +2 -1
- data/lib/llm/google_palm.rb +85 -0
- data/lib/loaders/html.rb +38 -0
- data/lib/prompt/base.rb +4 -2
- data/lib/prompt/prompt_template.rb +2 -2
- data/lib/version.rb +1 -1
- metadata +36 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff7f99d961b09e827df297ddb3144821c9103fd40eabb32688ca92588a73415c
|
4
|
+
data.tar.gz: bb83eaa99055cf45cceaccb18a84e9fd4ee3ea4a93a6a0c66e04ede43e5d4bc0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 40e5362520220d3ffc1b4c29c3e430b051de334c2f281d9cb7d7549a93be40b26b379dbd35d0c91ccb5010c1a495a653e31768f1b7a95bc087059d59339fd1a7
|
7
|
+
data.tar.gz: 04f24944b590ee8b577419a95718ad6796bc4cdc34d52cf05e287806912c05ba8ace22e07181a8537892124881b65a8a221e1c75d8c6245231dd5660c6b4308c
|
data/.env.example
CHANGED
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
@@ -10,7 +10,3 @@ gem "rake", "~> 13.0"
|
|
10
10
|
gem "rspec", "~> 3.0"
|
11
11
|
|
12
12
|
gem "standardrb"
|
13
|
-
|
14
|
-
# TODO: Fix this `faraday` issue where some gems are using 1.x and others are using 2.x
|
15
|
-
# Most likely everything will just need to be updated to `faraday 2.x`
|
16
|
-
gem "replicate-ruby", git: "https://github.com/andreibondarev/replicate-ruby.git", branch: "faraday-1.x"
|
data/Gemfile.lock
CHANGED
@@ -1,18 +1,7 @@
|
|
1
|
-
GIT
|
2
|
-
remote: https://github.com/andreibondarev/replicate-ruby.git
|
3
|
-
revision: 6276dec02ce33ae68a57cdd362eb8e87ed29c8e6
|
4
|
-
branch: faraday-1.x
|
5
|
-
specs:
|
6
|
-
replicate-ruby (0.2.1)
|
7
|
-
addressable
|
8
|
-
faraday (>= 1.0)
|
9
|
-
faraday-multipart
|
10
|
-
faraday-retry
|
11
|
-
|
12
1
|
PATH
|
13
2
|
remote: .
|
14
3
|
specs:
|
15
|
-
langchainrb (0.3.
|
4
|
+
langchainrb (0.3.11)
|
16
5
|
|
17
6
|
GEM
|
18
7
|
remote: https://rubygems.org/
|
@@ -128,6 +117,9 @@ GEM
|
|
128
117
|
faraday-retry (1.0.3)
|
129
118
|
faraday_middleware (1.2.0)
|
130
119
|
faraday (~> 1.0)
|
120
|
+
google_palm_api (0.1.0)
|
121
|
+
faraday (>= 1.0.0)
|
122
|
+
faraday_middleware (>= 1.0.0)
|
131
123
|
google_search_results (2.0.1)
|
132
124
|
graphlient (0.6.0)
|
133
125
|
faraday (>= 1.0)
|
@@ -208,6 +200,11 @@ GEM
|
|
208
200
|
rainbow (3.1.1)
|
209
201
|
rake (13.0.6)
|
210
202
|
regexp_parser (2.8.0)
|
203
|
+
replicate-ruby (0.2.2)
|
204
|
+
addressable
|
205
|
+
faraday (>= 1.0)
|
206
|
+
faraday-multipart
|
207
|
+
faraday-retry
|
211
208
|
rexml (3.2.5)
|
212
209
|
rspec (3.12.0)
|
213
210
|
rspec-core (~> 3.12.0)
|
@@ -285,16 +282,18 @@ DEPENDENCIES
|
|
285
282
|
docx (~> 0.8.0)
|
286
283
|
dotenv-rails (~> 2.7.6)
|
287
284
|
eqn (~> 1.6.5)
|
285
|
+
google_palm_api (~> 0.1.0)
|
288
286
|
google_search_results (~> 2.0.0)
|
289
287
|
hugging-face (~> 0.3.4)
|
290
288
|
langchainrb!
|
291
289
|
milvus (~> 0.9.0)
|
290
|
+
nokogiri (~> 1.13)
|
292
291
|
pdf-reader (~> 1.4)
|
293
292
|
pinecone (~> 0.1.6)
|
294
293
|
pry-byebug (~> 3.10.0)
|
295
294
|
qdrant-ruby (~> 0.9.0)
|
296
295
|
rake (~> 13.0)
|
297
|
-
replicate-ruby
|
296
|
+
replicate-ruby (~> 0.2.2)
|
298
297
|
rspec (~> 3.0)
|
299
298
|
ruby-openai (~> 4.0.0)
|
300
299
|
standardrb
|
data/README.md
CHANGED
@@ -136,11 +136,17 @@ cohere = LLM::HuggingFace.new(api_key: ENV["HUGGING_FACE_API_KEY"])
|
|
136
136
|
```
|
137
137
|
|
138
138
|
#### Replicate
|
139
|
-
Add `gem "replicate-ruby",
|
139
|
+
Add `gem "replicate-ruby", "~> 0.2.2"` to your Gemfile.
|
140
140
|
```ruby
|
141
141
|
cohere = LLM::Replicate.new(api_key: ENV["REPLICATE_API_KEY"])
|
142
142
|
```
|
143
143
|
|
144
|
+
#### Google PaLM (Pathways Language Model)
|
145
|
+
Add `"google_palm_api", "~> 0.1.0"` to your Gemfile.
|
146
|
+
```ruby
|
147
|
+
google_palm = LLM::GooglePalm.new(api_key: ENV["GOOGLE_PALM_API_KEY"])
|
148
|
+
```
|
149
|
+
|
144
150
|
### Using Prompts 📋
|
145
151
|
|
146
152
|
#### Prompt Templates
|
@@ -264,7 +270,8 @@ Need to read data from various sources? Load it up.
|
|
264
270
|
|
265
271
|
| Name | Class | Gem Requirements |
|
266
272
|
| ---- | ------------- | :--------------------------: |
|
267
|
-
| docx | Loaders::Docx | `gem "docx",
|
273
|
+
| docx | Loaders::Docx | `gem "docx", "~> 0.8.0"` |
|
274
|
+
| html | Loaders::HTML | `gem "nokogiri", "~> 1.13"` |
|
268
275
|
| pdf | Loaders::PDF | `gem "pdf-reader", "~> 1.4"` |
|
269
276
|
| text | Loaders::Text | |
|
270
277
|
|
@@ -284,7 +291,7 @@ Langchain.logger.level = :info
|
|
284
291
|
|
285
292
|
1. `git clone https://github.com/andreibondarev/langchainrb.git`
|
286
293
|
2. `cp .env.example .env`, then fill out the environment variables in `.env`
|
287
|
-
3. `
|
294
|
+
3. `bundle exec rake` to ensure that the tests pass and to run standardrb
|
288
295
|
4. `bin/console` to load the gem in a REPL session. Feel free to add your own instances of LLMs, Tools, Agents, etc. and experiment with them.
|
289
296
|
|
290
297
|
## Core Contributors
|
data/lib/dependency_helper.rb
CHANGED
@@ -1,12 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
VersionError = Class.new(ScriptError)
|
4
|
+
|
3
5
|
# This method requires and loads the given gem, and then checks to see if the version of the gem meets the requirements listed in `langchain.gemspec`
|
4
6
|
# This solution was built to avoid auto-loading every single gem in the Gemfile when the developer will mostly likely be only using a few of them.
|
5
7
|
#
|
6
8
|
# @param gem_name [String] The name of the gem to load
|
7
9
|
# @return [Boolean] Whether or not the gem was loaded successfully
|
8
10
|
# @raise [LoadError] If the gem is not installed
|
9
|
-
# @raise [
|
11
|
+
# @raise [VersionError] If the gem is installed, but the version does not meet the requirements
|
10
12
|
#
|
11
13
|
def depends_on(gem_name)
|
12
14
|
gem(gem_name) # require the gem
|
@@ -14,10 +16,12 @@ def depends_on(gem_name)
|
|
14
16
|
return(true) unless defined?(Bundler) # If we're in a non-bundler environment, we're no longer able to determine if we'll meet requirements
|
15
17
|
|
16
18
|
gem_version = Gem.loaded_specs[gem_name].version
|
17
|
-
gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }
|
19
|
+
gem_requirement = Bundler.load.dependencies.find { |g| g.name == gem_name }&.requirement
|
20
|
+
|
21
|
+
raise LoadError unless gem_requirement
|
18
22
|
|
19
|
-
|
20
|
-
raise "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
|
23
|
+
unless gem_requirement.satisfied_by?(gem_version)
|
24
|
+
raise VersionError, "The #{gem_name} gem is installed, but version #{gem_requirement} is required. You have #{gem_version}."
|
21
25
|
end
|
22
26
|
|
23
27
|
true
|
data/lib/langchain.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "logger"
|
4
|
+
require "pathname"
|
4
5
|
|
5
6
|
require_relative "./version"
|
6
7
|
require_relative "./dependency_helper"
|
@@ -34,6 +35,7 @@ end
|
|
34
35
|
module LLM
|
35
36
|
autoload :Base, "llm/base"
|
36
37
|
autoload :Cohere, "llm/cohere"
|
38
|
+
autoload :GooglePalm, "llm/google_palm"
|
37
39
|
autoload :HuggingFace, "llm/hugging_face"
|
38
40
|
autoload :OpenAI, "llm/openai"
|
39
41
|
autoload :Replicate, "llm/replicate"
|
@@ -59,6 +61,7 @@ module Loaders
|
|
59
61
|
autoload :Docx, "loaders/docx"
|
60
62
|
autoload :PDF, "loaders/pdf"
|
61
63
|
autoload :Text, "loaders/text"
|
64
|
+
autoload :HTML, "loaders/html"
|
62
65
|
end
|
63
66
|
|
64
67
|
autoload :Loader, "loader"
|
data/lib/llm/base.rb
CHANGED
@@ -7,9 +7,10 @@ module LLM
|
|
7
7
|
# Currently supported LLMs
|
8
8
|
# TODO: Add support for HuggingFace and other LLMs
|
9
9
|
LLMS = {
|
10
|
-
openai: "OpenAI",
|
11
10
|
cohere: "Cohere",
|
11
|
+
google_palm: "GooglePalm",
|
12
12
|
huggingface: "HuggingFace",
|
13
|
+
openai: "OpenAI",
|
13
14
|
replicate: "Replicate"
|
14
15
|
}.freeze
|
15
16
|
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module LLM
|
4
|
+
class GooglePalm < Base
|
5
|
+
# Wrapper around the Google PaLM (Pathways Language Model) APIs.
|
6
|
+
|
7
|
+
DEFAULTS = {
|
8
|
+
temperature: 0.0,
|
9
|
+
dimension: 768 # This is what the `embedding-gecko-001` model generates
|
10
|
+
}.freeze
|
11
|
+
|
12
|
+
def initialize(api_key:)
|
13
|
+
depends_on "google_palm_api"
|
14
|
+
require "google_palm_api"
|
15
|
+
|
16
|
+
@client = ::GooglePalmApi::Client.new(api_key: api_key)
|
17
|
+
end
|
18
|
+
|
19
|
+
#
|
20
|
+
# Generate an embedding for a given text
|
21
|
+
#
|
22
|
+
# @param text [String] The text to generate an embedding for
|
23
|
+
# @return [Array] The embedding
|
24
|
+
#
|
25
|
+
def embed(text:)
|
26
|
+
response = client.embed(
|
27
|
+
text: text
|
28
|
+
)
|
29
|
+
response.dig("embedding", "value")
|
30
|
+
end
|
31
|
+
|
32
|
+
#
|
33
|
+
# Generate a completion for a given prompt
|
34
|
+
#
|
35
|
+
# @param prompt [String] The prompt to generate a completion for
|
36
|
+
# @return [String] The completion
|
37
|
+
#
|
38
|
+
def complete(prompt:, **params)
|
39
|
+
default_params = {
|
40
|
+
prompt: prompt,
|
41
|
+
temperature: DEFAULTS[:temperature]
|
42
|
+
}
|
43
|
+
|
44
|
+
if params[:stop_sequences]
|
45
|
+
default_params[:stop_sequences] = params.delete(:stop_sequences)
|
46
|
+
end
|
47
|
+
|
48
|
+
if params[:max_tokens]
|
49
|
+
default_params[:max_output_tokens] = params.delete(:max_tokens)
|
50
|
+
end
|
51
|
+
|
52
|
+
default_params.merge!(params)
|
53
|
+
|
54
|
+
response = client.generate_text(**default_params)
|
55
|
+
response.dig("candidates", 0, "output")
|
56
|
+
end
|
57
|
+
|
58
|
+
#
|
59
|
+
# Generate a chat completion for a given prompt
|
60
|
+
#
|
61
|
+
# @param prompt [String] The prompt to generate a chat completion for
|
62
|
+
# @return [String] The chat completion
|
63
|
+
#
|
64
|
+
def chat(prompt:, **params)
|
65
|
+
# TODO: Figure out how to introduce persisted conversations
|
66
|
+
default_params = {
|
67
|
+
prompt: prompt,
|
68
|
+
temperature: DEFAULTS[:temperature]
|
69
|
+
}
|
70
|
+
|
71
|
+
if params[:stop_sequences]
|
72
|
+
default_params[:stop] = params.delete(:stop_sequences)
|
73
|
+
end
|
74
|
+
|
75
|
+
if params[:max_tokens]
|
76
|
+
default_params[:max_output_tokens] = params.delete(:max_tokens)
|
77
|
+
end
|
78
|
+
|
79
|
+
default_params.merge!(params)
|
80
|
+
|
81
|
+
response = client.generate_chat_message(**default_params)
|
82
|
+
response.dig("candidates", 0, "content")
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
data/lib/loaders/html.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "open-uri"
|
4
|
+
|
5
|
+
module Loaders
|
6
|
+
class HTML < Base
|
7
|
+
# We only look for headings and paragraphs
|
8
|
+
TEXT_CONTENT_TAGS = %w[h1 h2 h3 h4 h5 h6 p]
|
9
|
+
|
10
|
+
#
|
11
|
+
# This Loader parses URL into a text.
|
12
|
+
# If you'd like to use it directly you can do so like this:
|
13
|
+
# Loaders::URL.new("https://nokogiri.org/").load
|
14
|
+
#
|
15
|
+
def initialize(url)
|
16
|
+
depends_on "nokogiri"
|
17
|
+
require "nokogiri"
|
18
|
+
|
19
|
+
@url = url
|
20
|
+
end
|
21
|
+
|
22
|
+
# Check that url is a valid URL
|
23
|
+
def loadable?
|
24
|
+
!!(@url =~ URI::DEFAULT_PARSER.make_regexp)
|
25
|
+
end
|
26
|
+
|
27
|
+
def load
|
28
|
+
return unless response.status.first == "200"
|
29
|
+
|
30
|
+
doc = Nokogiri::HTML(response.read)
|
31
|
+
doc.css(TEXT_CONTENT_TAGS.join(",")).map(&:inner_text).join("\n\n")
|
32
|
+
end
|
33
|
+
|
34
|
+
def response
|
35
|
+
@response ||= URI.parse(@url).open
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/prompt/base.rb
CHANGED
@@ -66,6 +66,8 @@ module Prompt
|
|
66
66
|
# contained within the template. Input variables are defined as text enclosed in
|
67
67
|
# curly braces (e.g. "{variable_name}").
|
68
68
|
#
|
69
|
+
# Content within two consecutive curly braces (e.g. "{{ignore_me}}) are ignored.
|
70
|
+
#
|
69
71
|
# @param template [String] The template string to extract variables from.
|
70
72
|
#
|
71
73
|
# @return [Array<String>] An array of input variable names.
|
@@ -74,9 +76,9 @@ module Prompt
|
|
74
76
|
input_variables = []
|
75
77
|
scanner = StringScanner.new(template)
|
76
78
|
|
77
|
-
while scanner.scan_until(/\{([^
|
79
|
+
while scanner.scan_until(/\{([^}]*)\}/)
|
78
80
|
variable = scanner[1].strip
|
79
|
-
input_variables << variable unless variable.empty?
|
81
|
+
input_variables << variable unless variable.empty? || variable[0] == "{"
|
80
82
|
end
|
81
83
|
|
82
84
|
input_variables
|
@@ -20,7 +20,7 @@ module Prompt
|
|
20
20
|
end
|
21
21
|
|
22
22
|
#
|
23
|
-
# Format the prompt with the inputs.
|
23
|
+
# Format the prompt with the inputs. Double {{}} replaced with single {} to adhere to f-string spec.
|
24
24
|
#
|
25
25
|
# @param kwargs [Hash] Any arguments to be passed to the prompt template.
|
26
26
|
# @return [String] A formatted string.
|
@@ -28,7 +28,7 @@ module Prompt
|
|
28
28
|
def format(**kwargs)
|
29
29
|
result = @template
|
30
30
|
kwargs.each { |key, value| result = result.gsub(/\{#{key}\}/, value.to_s) }
|
31
|
-
result
|
31
|
+
result.gsub(/{{/, "{").gsub(/}}/, "}")
|
32
32
|
end
|
33
33
|
|
34
34
|
#
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: langchainrb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrei Bondarev
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-05-
|
11
|
+
date: 2023-05-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dotenv-rails
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 1.6.5
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: google_palm_api
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.1.0
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 0.1.0
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: google_search_results
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,6 +150,20 @@ dependencies:
|
|
136
150
|
- - "~>"
|
137
151
|
- !ruby/object:Gem::Version
|
138
152
|
version: 0.9.0
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: nokogiri
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '1.13'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '1.13'
|
139
167
|
- !ruby/object:Gem::Dependency
|
140
168
|
name: pdf-reader
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -168,16 +196,16 @@ dependencies:
|
|
168
196
|
name: replicate-ruby
|
169
197
|
requirement: !ruby/object:Gem::Requirement
|
170
198
|
requirements:
|
171
|
-
- - "
|
199
|
+
- - "~>"
|
172
200
|
- !ruby/object:Gem::Version
|
173
|
-
version:
|
201
|
+
version: 0.2.2
|
174
202
|
type: :development
|
175
203
|
prerelease: false
|
176
204
|
version_requirements: !ruby/object:Gem::Requirement
|
177
205
|
requirements:
|
178
|
-
- - "
|
206
|
+
- - "~>"
|
179
207
|
- !ruby/object:Gem::Version
|
180
|
-
version:
|
208
|
+
version: 0.2.2
|
181
209
|
- !ruby/object:Gem::Dependency
|
182
210
|
name: qdrant-ruby
|
183
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -262,12 +290,14 @@ files:
|
|
262
290
|
- lib/langchain.rb
|
263
291
|
- lib/llm/base.rb
|
264
292
|
- lib/llm/cohere.rb
|
293
|
+
- lib/llm/google_palm.rb
|
265
294
|
- lib/llm/hugging_face.rb
|
266
295
|
- lib/llm/openai.rb
|
267
296
|
- lib/llm/replicate.rb
|
268
297
|
- lib/loader.rb
|
269
298
|
- lib/loaders/base.rb
|
270
299
|
- lib/loaders/docx.rb
|
300
|
+
- lib/loaders/html.rb
|
271
301
|
- lib/loaders/pdf.rb
|
272
302
|
- lib/loaders/text.rb
|
273
303
|
- lib/prompt/base.rb
|