langchainrb 0.3.13 → 0.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/Gemfile.lock +1 -1
- data/README.md +1 -0
- data/lib/langchain/data.rb +16 -0
- data/lib/langchain/loader.rb +4 -3
- data/lib/langchain.rb +1 -0
- data/lib/llm/base.rb +5 -0
- data/lib/llm/cohere.rb +17 -0
- data/lib/llm/google_palm.rb +20 -0
- data/lib/llm/hugging_face.rb +4 -0
- data/lib/llm/openai.rb +29 -0
- data/lib/llm/prompts/summarize_template.json +5 -0
- data/lib/llm/replicate.rb +29 -0
- data/lib/vectorsearch/base.rb +1 -1
- data/lib/vectorsearch/weaviate.rb +13 -36
- data/lib/version.rb +1 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 33c9436ac8d6a73dc06d30f63c11e4f246b3705aa8934765a53ee59325c3a9cd
|
4
|
+
data.tar.gz: 9cc85603694f9367dd162e25379029a345aa0b5c88cccf303c2af114d43a4010
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ca5e81638625939d11999a64d44c92fc57c762a934aa8fd5b110c3f5aacc9a736ab5f02da4366e7a1b9b9ec0335dd1eb1683f5b9d90bd97c81914ea0a698dc7c
|
7
|
+
data.tar.gz: aff49ef9451bcbc9a97d181757a5b913737cbfbb4fc3ca49d423cbd2e59a4a71091816e98c2996ff7f8292cb6e0c0d69931a4f3e4e36e2a69fdd7f745640e266
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
+
## [0.3.14] - 2023-05-28
|
4
|
+
- 🔍 Vectorsearch
|
5
|
+
- Not relying on Weaviate modules anymore
|
6
|
+
- Adding missing specs for Qdrant and Milvus classes
|
7
|
+
- 🚚 Loaders
|
8
|
+
- Add Langchain::Data result object for data loaders
|
9
|
+
- 🗣️ LLMs
|
10
|
+
- Add `summarize()` method to the LLMs
|
11
|
+
|
3
12
|
## [0.3.13] - 2023-05-26
|
4
13
|
- 🔍 Vectorsearch
|
5
14
|
- Pgvector support
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -321,6 +321,7 @@ Langchain.logger.level = :info
|
|
321
321
|
[<img style="border-radius:50%" alt="Andrei Bondarev" src="https://avatars.githubusercontent.com/u/541665?v=4" width="80" height="80" class="avatar">](https://github.com/andreibondarev)
|
322
322
|
[<img style="border-radius:50%" alt="Rafael Figueiredo" src="https://avatars.githubusercontent.com/u/35845775?v=4" width="80" height="80" class="avatar">](https://github.com/rafaelqfigueiredo)
|
323
323
|
[<img style="border-radius:50%" alt="Ricky Chilcott" src="https://avatars.githubusercontent.com/u/445759?v=4" width="80" height="80" class="avatar">](https://github.com/rickychilcott)
|
324
|
+
[<img style="border-radius:50%" alt="Alex Chaplinsky" src="https://avatars.githubusercontent.com/u/695947?v=4" width="80" height="80" class="avatar">](https://github.com/alchaplinsky)
|
324
325
|
|
325
326
|
(Criteria for becoming an Honorary Contributor or Core Contributor is pending...)
|
326
327
|
|
data/lib/langchain/loader.rb
CHANGED
@@ -58,11 +58,12 @@ module Langchain
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def process(&block)
|
61
|
-
|
61
|
+
raw_data, kind = yield
|
62
62
|
|
63
|
-
raise UnknownFormatError unless
|
63
|
+
raise UnknownFormatError unless kind
|
64
64
|
|
65
|
-
Langchain::Processors.const_get(
|
65
|
+
processor = Langchain::Processors.const_get(kind).new
|
66
|
+
Langchain::Data.new(processor.parse(raw_data), source: @path)
|
66
67
|
end
|
67
68
|
|
68
69
|
def find_processor(constant, value)
|
data/lib/langchain.rb
CHANGED
data/lib/llm/base.rb
CHANGED
@@ -33,6 +33,11 @@ module LLM
|
|
33
33
|
raise NotImplementedError, "#{self.class.name} does not support generating embeddings"
|
34
34
|
end
|
35
35
|
|
36
|
+
# Method supported by an LLM that summarizes a given text
|
37
|
+
def summarize(...)
|
38
|
+
raise NotImplementedError, "#{self.class.name} does not support summarization"
|
39
|
+
end
|
40
|
+
|
36
41
|
# Ensure that the LLM value passed in is supported
|
37
42
|
# @param llm [Symbol] The LLM to use
|
38
43
|
def self.validate_llm!(llm:)
|
data/lib/llm/cohere.rb
CHANGED
@@ -16,9 +16,12 @@ module LLM
|
|
16
16
|
@client = ::Cohere::Client.new(api_key: api_key)
|
17
17
|
end
|
18
18
|
|
19
|
+
#
|
19
20
|
# Generate an embedding for a given text
|
21
|
+
#
|
20
22
|
# @param text [String] The text to generate an embedding for
|
21
23
|
# @return [Hash] The embedding
|
24
|
+
#
|
22
25
|
def embed(text:)
|
23
26
|
response = client.embed(
|
24
27
|
texts: [text],
|
@@ -27,9 +30,12 @@ module LLM
|
|
27
30
|
response.dig("embeddings").first
|
28
31
|
end
|
29
32
|
|
33
|
+
#
|
30
34
|
# Generate a completion for a given prompt
|
35
|
+
#
|
31
36
|
# @param prompt [String] The prompt to generate a completion for
|
32
37
|
# @return [Hash] The completion
|
38
|
+
#
|
33
39
|
def complete(prompt:, **params)
|
34
40
|
default_params = {
|
35
41
|
prompt: prompt,
|
@@ -51,5 +57,16 @@ module LLM
|
|
51
57
|
def chat(...)
|
52
58
|
complete(...)
|
53
59
|
end
|
60
|
+
|
61
|
+
# Generate a summary in English for a given text
|
62
|
+
#
|
63
|
+
# More parameters available to extend this method with: https://github.com/andreibondarev/cohere-ruby/blob/0.9.4/lib/cohere/client.rb#L107-L115
|
64
|
+
#
|
65
|
+
# @param text [String] The text to generate a summary for
|
66
|
+
# @return [String] The summary
|
67
|
+
def summarize(text:)
|
68
|
+
response = client.summarize(text: text)
|
69
|
+
response.dig("summary")
|
70
|
+
end
|
54
71
|
end
|
55
72
|
end
|
data/lib/llm/google_palm.rb
CHANGED
@@ -81,5 +81,25 @@ module LLM
|
|
81
81
|
response = client.generate_chat_message(**default_params)
|
82
82
|
response.dig("candidates", 0, "content")
|
83
83
|
end
|
84
|
+
|
85
|
+
#
|
86
|
+
# Generate a summarization for a given text
|
87
|
+
#
|
88
|
+
# @param text [String] The text to generate a summarization for
|
89
|
+
# @return [String] The summarization
|
90
|
+
#
|
91
|
+
def summarize(text:)
|
92
|
+
prompt_template = Prompt.load_from_path(
|
93
|
+
file_path: Langchain.root.join("llm/prompts/summarize_template.json")
|
94
|
+
)
|
95
|
+
prompt = prompt_template.format(text: text)
|
96
|
+
|
97
|
+
complete(
|
98
|
+
prompt: prompt,
|
99
|
+
temperature: DEFAULTS[:temperature],
|
100
|
+
# Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
|
101
|
+
max_tokens: 2048
|
102
|
+
)
|
103
|
+
end
|
84
104
|
end
|
85
105
|
end
|
data/lib/llm/hugging_face.rb
CHANGED
@@ -12,6 +12,7 @@ module LLM
|
|
12
12
|
|
13
13
|
#
|
14
14
|
# Intialize the HuggingFace LLM
|
15
|
+
#
|
15
16
|
# @param api_key [String] The API key to use
|
16
17
|
#
|
17
18
|
def initialize(api_key:)
|
@@ -21,9 +22,12 @@ module LLM
|
|
21
22
|
@client = ::HuggingFace::InferenceApi.new(api_token: api_key)
|
22
23
|
end
|
23
24
|
|
25
|
+
#
|
24
26
|
# Generate an embedding for a given text
|
27
|
+
#
|
25
28
|
# @param text [String] The text to embed
|
26
29
|
# @return [Array] The embedding
|
30
|
+
#
|
27
31
|
def embed(text:)
|
28
32
|
client.embedding(
|
29
33
|
input: text,
|
data/lib/llm/openai.rb
CHANGED
@@ -18,9 +18,12 @@ module LLM
|
|
18
18
|
@client = ::OpenAI::Client.new(access_token: api_key)
|
19
19
|
end
|
20
20
|
|
21
|
+
#
|
21
22
|
# Generate an embedding for a given text
|
23
|
+
#
|
22
24
|
# @param text [String] The text to generate an embedding for
|
23
25
|
# @return [Array] The embedding
|
26
|
+
#
|
24
27
|
def embed(text:)
|
25
28
|
response = client.embeddings(
|
26
29
|
parameters: {
|
@@ -31,9 +34,12 @@ module LLM
|
|
31
34
|
response.dig("data").first.dig("embedding")
|
32
35
|
end
|
33
36
|
|
37
|
+
#
|
34
38
|
# Generate a completion for a given prompt
|
39
|
+
#
|
35
40
|
# @param prompt [String] The prompt to generate a completion for
|
36
41
|
# @return [String] The completion
|
42
|
+
#
|
37
43
|
def complete(prompt:, **params)
|
38
44
|
default_params = {
|
39
45
|
model: DEFAULTS[:completion_model_name],
|
@@ -51,9 +57,12 @@ module LLM
|
|
51
57
|
response.dig("choices", 0, "text")
|
52
58
|
end
|
53
59
|
|
60
|
+
#
|
54
61
|
# Generate a chat completion for a given prompt
|
62
|
+
#
|
55
63
|
# @param prompt [String] The prompt to generate a chat completion for
|
56
64
|
# @return [String] The chat completion
|
65
|
+
#
|
57
66
|
def chat(prompt:, **params)
|
58
67
|
default_params = {
|
59
68
|
model: DEFAULTS[:chat_completion_model_name],
|
@@ -71,5 +80,25 @@ module LLM
|
|
71
80
|
response = client.chat(parameters: default_params)
|
72
81
|
response.dig("choices", 0, "message", "content")
|
73
82
|
end
|
83
|
+
|
84
|
+
#
|
85
|
+
# Generate a summary for a given text
|
86
|
+
#
|
87
|
+
# @param text [String] The text to generate a summary for
|
88
|
+
# @return [String] The summary
|
89
|
+
#
|
90
|
+
def summarize(text:)
|
91
|
+
prompt_template = Prompt.load_from_path(
|
92
|
+
file_path: Langchain.root.join("llm/prompts/summarize_template.json")
|
93
|
+
)
|
94
|
+
prompt = prompt_template.format(text: text)
|
95
|
+
|
96
|
+
complete(
|
97
|
+
prompt: prompt,
|
98
|
+
temperature: DEFAULTS[:temperature],
|
99
|
+
# Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
|
100
|
+
max_tokens: 2048
|
101
|
+
)
|
102
|
+
end
|
74
103
|
end
|
75
104
|
end
|
data/lib/llm/replicate.rb
CHANGED
@@ -23,8 +23,11 @@ module LLM
|
|
23
23
|
dimension: 384
|
24
24
|
}.freeze
|
25
25
|
|
26
|
+
#
|
26
27
|
# Intialize the Replicate LLM
|
28
|
+
#
|
27
29
|
# @param api_key [String] The API key to use
|
30
|
+
#
|
28
31
|
def initialize(api_key:)
|
29
32
|
depends_on "replicate-ruby"
|
30
33
|
require "replicate"
|
@@ -36,9 +39,12 @@ module LLM
|
|
36
39
|
@client = ::Replicate.client
|
37
40
|
end
|
38
41
|
|
42
|
+
#
|
39
43
|
# Generate an embedding for a given text
|
44
|
+
#
|
40
45
|
# @param text [String] The text to generate an embedding for
|
41
46
|
# @return [Hash] The embedding
|
47
|
+
#
|
42
48
|
def embed(text:)
|
43
49
|
response = embeddings_model.predict(input: text)
|
44
50
|
|
@@ -50,9 +56,12 @@ module LLM
|
|
50
56
|
response.output
|
51
57
|
end
|
52
58
|
|
59
|
+
#
|
53
60
|
# Generate a completion for a given prompt
|
61
|
+
#
|
54
62
|
# @param prompt [String] The prompt to generate a completion for
|
55
63
|
# @return [Hash] The completion
|
64
|
+
#
|
56
65
|
def complete(prompt:, **params)
|
57
66
|
response = completion_model.predict(prompt: prompt)
|
58
67
|
|
@@ -73,6 +82,26 @@ module LLM
|
|
73
82
|
complete(...)
|
74
83
|
end
|
75
84
|
|
85
|
+
#
|
86
|
+
# Generate a summary for a given text
|
87
|
+
#
|
88
|
+
# @param text [String] The text to generate a summary for
|
89
|
+
# @return [String] The summary
|
90
|
+
#
|
91
|
+
def summarize(text:)
|
92
|
+
prompt_template = Prompt.load_from_path(
|
93
|
+
file_path: Langchain.root.join("llm/prompts/summarize_template.json")
|
94
|
+
)
|
95
|
+
prompt = prompt_template.format(text: text)
|
96
|
+
|
97
|
+
complete(
|
98
|
+
prompt: prompt,
|
99
|
+
temperature: DEFAULTS[:temperature],
|
100
|
+
# Most models have a context length of 2048 tokens (except for the newest models, which support 4096).
|
101
|
+
max_tokens: 2048
|
102
|
+
)
|
103
|
+
end
|
104
|
+
|
76
105
|
alias_method :generate_embedding, :embed
|
77
106
|
|
78
107
|
private
|
data/lib/vectorsearch/base.rb
CHANGED
@@ -14,9 +14,7 @@ module Vectorsearch
|
|
14
14
|
|
15
15
|
@client = ::Weaviate::Client.new(
|
16
16
|
url: url,
|
17
|
-
api_key: api_key
|
18
|
-
model_service: llm,
|
19
|
-
model_service_api_key: llm_api_key
|
17
|
+
api_key: api_key
|
20
18
|
)
|
21
19
|
@index_name = index_name
|
22
20
|
|
@@ -30,7 +28,8 @@ module Vectorsearch
|
|
30
28
|
objects = Array(texts).map do |text|
|
31
29
|
{
|
32
30
|
class: index_name,
|
33
|
-
properties: {content: text}
|
31
|
+
properties: {content: text},
|
32
|
+
vector: llm_client.embed(text: text)
|
34
33
|
}
|
35
34
|
end
|
36
35
|
|
@@ -43,11 +42,7 @@ module Vectorsearch
|
|
43
42
|
def create_default_schema
|
44
43
|
client.schema.create(
|
45
44
|
class_name: index_name,
|
46
|
-
vectorizer: "
|
47
|
-
# TODO: Figure out a way to optionally enable it
|
48
|
-
# "module_config": {
|
49
|
-
# "qna-openai": {}
|
50
|
-
# },
|
45
|
+
vectorizer: "none",
|
51
46
|
properties: [
|
52
47
|
# TODO: Allow passing in your own IDs
|
53
48
|
{
|
@@ -63,14 +58,9 @@ module Vectorsearch
|
|
63
58
|
# @param k [Integer|String] The number of results to return
|
64
59
|
# @return [Hash] The search results
|
65
60
|
def similarity_search(query:, k: 4)
|
66
|
-
|
61
|
+
embedding = llm_client.embed(text: query)
|
67
62
|
|
68
|
-
|
69
|
-
class_name: index_name,
|
70
|
-
near_text: near_text,
|
71
|
-
limit: k.to_s,
|
72
|
-
fields: "content _additional { id }"
|
73
|
-
)
|
63
|
+
similarity_search_by_vector(embedding: embedding, k: k)
|
74
64
|
end
|
75
65
|
|
76
66
|
# Return documents similar to the vector
|
@@ -92,29 +82,16 @@ module Vectorsearch
|
|
92
82
|
# @param question [String] The question to ask
|
93
83
|
# @return [Hash] The answer
|
94
84
|
def ask(question:)
|
95
|
-
|
96
|
-
# The Cohere support is on the way: https://github.com/weaviate/weaviate/pull/2600
|
97
|
-
if llm == :openai
|
98
|
-
ask_object = "{ question: \"#{question}\" }"
|
99
|
-
|
100
|
-
client.query.get(
|
101
|
-
class_name: index_name,
|
102
|
-
ask: ask_object,
|
103
|
-
limit: "1",
|
104
|
-
fields: "_additional { answer { result } }"
|
105
|
-
)
|
106
|
-
elsif llm == :cohere
|
107
|
-
search_results = similarity_search(query: question)
|
85
|
+
search_results = similarity_search(query: question)
|
108
86
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
87
|
+
context = search_results.map do |result|
|
88
|
+
result.dig("content").to_s
|
89
|
+
end
|
90
|
+
context = context.join("\n---\n")
|
113
91
|
|
114
|
-
|
92
|
+
prompt = generate_prompt(question: question, context: context)
|
115
93
|
|
116
|
-
|
117
|
-
end
|
94
|
+
llm_client.chat(prompt: prompt)
|
118
95
|
end
|
119
96
|
end
|
120
97
|
end
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: langchainrb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrei Bondarev
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-05-
|
11
|
+
date: 2023-05-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dotenv-rails
|
@@ -316,6 +316,7 @@ files:
|
|
316
316
|
- lib/agent/chain_of_thought_agent/chain_of_thought_agent_prompt.json
|
317
317
|
- lib/dependency_helper.rb
|
318
318
|
- lib/langchain.rb
|
319
|
+
- lib/langchain/data.rb
|
319
320
|
- lib/langchain/loader.rb
|
320
321
|
- lib/langchain/processors/base.rb
|
321
322
|
- lib/langchain/processors/csv.rb
|
@@ -331,6 +332,7 @@ files:
|
|
331
332
|
- lib/llm/google_palm.rb
|
332
333
|
- lib/llm/hugging_face.rb
|
333
334
|
- lib/llm/openai.rb
|
335
|
+
- lib/llm/prompts/summarize_template.json
|
334
336
|
- lib/llm/replicate.rb
|
335
337
|
- lib/prompt/base.rb
|
336
338
|
- lib/prompt/few_shot_prompt_template.rb
|