boxcars 0.6.3 → 0.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/Gemfile.lock +1 -1
- data/lib/boxcars/engine/cohere.rb +0 -49
- data/lib/boxcars/engine/groq.rb +15 -68
- data/lib/boxcars/engine/ollama.rb +80 -0
- data/lib/boxcars/engine/openai.rb +0 -53
- data/lib/boxcars/engine/perplexityai.rb +21 -92
- data/lib/boxcars/engine.rb +54 -0
- data/lib/boxcars/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c7288e4aaea4065096105c84b78e9425330847eba37b55b6479466c55ae840a8
|
|
4
|
+
data.tar.gz: 6ff9ec0d0c69113dc70360750fa8fe5460236401c75a3b4e681009efad733301
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e48e4debf178dca7647a08357eaabbdd8ceef6edc3ae40d49a25a31b8ee21fbc41ded409fd55c2b840762868546f1edb0ada72729178c15953e6f48cc0d832fe
|
|
7
|
+
data.tar.gz: 3717955095a68207dd7c0ee914132573d84126c01ea4e53cec0d5a46985f512a59da3fb1b78ededac29c0d72d16e5f70301629ab326aa20ec2feb4b6664ae343
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [v0.6.3](https://github.com/BoxcarsAI/boxcars/tree/v0.6.3) (2024-07-26)
|
|
4
|
+
|
|
5
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.6.2...v0.6.3)
|
|
6
|
+
|
|
7
|
+
**Merged pull requests:**
|
|
8
|
+
|
|
9
|
+
- Add Groq engine [\#199](https://github.com/BoxcarsAI/boxcars/pull/199) ([francis](https://github.com/francis))
|
|
10
|
+
|
|
11
|
+
## [v0.6.2](https://github.com/BoxcarsAI/boxcars/tree/v0.6.2) (2024-07-24)
|
|
12
|
+
|
|
13
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.6.1...v0.6.2)
|
|
14
|
+
|
|
15
|
+
**Merged pull requests:**
|
|
16
|
+
|
|
17
|
+
- add flag for symbolizing JSON Engine Boxcar results [\#198](https://github.com/BoxcarsAI/boxcars/pull/198) ([francis](https://github.com/francis))
|
|
18
|
+
|
|
3
19
|
## [v0.6.1](https://github.com/BoxcarsAI/boxcars/tree/v0.6.1) (2024-07-19)
|
|
4
20
|
|
|
5
21
|
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.5.1...v0.6.1)
|
data/Gemfile.lock
CHANGED
|
@@ -90,21 +90,6 @@ module Boxcars
|
|
|
90
90
|
llm_params
|
|
91
91
|
end
|
|
92
92
|
|
|
93
|
-
# Get generation informaton
|
|
94
|
-
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
95
|
-
# @return [Array<Generation>] The generation information.
|
|
96
|
-
def generation_info(sub_choices)
|
|
97
|
-
sub_choices.map do |choice|
|
|
98
|
-
Generation.new(
|
|
99
|
-
text: choice["completion"],
|
|
100
|
-
generation_info: {
|
|
101
|
-
finish_reason: choice.fetch("stop_reason", nil),
|
|
102
|
-
logprobs: choice.fetch("logprobs", nil)
|
|
103
|
-
}
|
|
104
|
-
)
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
93
|
# make sure we got a valid response
|
|
109
94
|
# @param response [Hash] The response to check.
|
|
110
95
|
# @param must_haves [Array<String>] The keys that must be in the response. Defaults to %w[choices].
|
|
@@ -124,45 +109,11 @@ module Boxcars
|
|
|
124
109
|
end
|
|
125
110
|
end
|
|
126
111
|
|
|
127
|
-
# Call out to OpenAI's endpoint with k unique prompts.
|
|
128
|
-
# @param prompts [Array<String>] The prompts to pass into the model.
|
|
129
|
-
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
|
130
|
-
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
131
|
-
# @return [EngineResult] The full engine output.
|
|
132
|
-
def generate(prompts:, stop: nil)
|
|
133
|
-
params = {}
|
|
134
|
-
params[:stop] = stop if stop
|
|
135
|
-
choices = []
|
|
136
|
-
# Get the token usage from the response.
|
|
137
|
-
# Includes prompt, completion, and total tokens used.
|
|
138
|
-
prompts.each_slice(batch_size) do |sub_prompts|
|
|
139
|
-
sub_prompts.each do |sprompts, inputs|
|
|
140
|
-
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
141
|
-
check_response(response)
|
|
142
|
-
choices << response
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
n = params.fetch(:n, 1)
|
|
147
|
-
generations = []
|
|
148
|
-
prompts.each_with_index do |_prompt, i|
|
|
149
|
-
sub_choices = choices[i * n, (i + 1) * n]
|
|
150
|
-
generations.push(generation_info(sub_choices))
|
|
151
|
-
end
|
|
152
|
-
EngineResult.new(generations: generations, engine_output: { token_usage: {} })
|
|
153
|
-
end
|
|
154
|
-
# rubocop:enable Metrics/AbcSize
|
|
155
|
-
|
|
156
112
|
# the engine type
|
|
157
113
|
def engine_type
|
|
158
114
|
"claude"
|
|
159
115
|
end
|
|
160
116
|
|
|
161
|
-
# calculate the number of tokens used
|
|
162
|
-
def get_num_tokens(text:)
|
|
163
|
-
text.split.length # TODO: hook up to token counting gem
|
|
164
|
-
end
|
|
165
|
-
|
|
166
117
|
# lookup the context size for a model by name
|
|
167
118
|
# @param modelname [String] The name of the model to lookup.
|
|
168
119
|
def modelname_to_contextsize(_modelname)
|
data/lib/boxcars/engine/groq.rb
CHANGED
|
@@ -71,8 +71,8 @@ module Boxcars
|
|
|
71
71
|
prompt = Prompt.new(template: question)
|
|
72
72
|
response = client(prompt: prompt, **kwargs)
|
|
73
73
|
raise Error, "Groq: No response from API" unless response
|
|
74
|
-
raise Error, "Groq: #{response['error']}" if response["error"]
|
|
75
74
|
|
|
75
|
+
check_response(response)
|
|
76
76
|
answer = response["choices"].map { |c| c.dig("message", "content") || c["text"] }.join("\n").strip
|
|
77
77
|
puts answer
|
|
78
78
|
answer
|
|
@@ -83,31 +83,16 @@ module Boxcars
|
|
|
83
83
|
groq_parmas
|
|
84
84
|
end
|
|
85
85
|
|
|
86
|
-
# Get generation informaton
|
|
87
|
-
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
88
|
-
# @return [Array<Generation>] The generation information.
|
|
89
|
-
def generation_info(sub_choices)
|
|
90
|
-
sub_choices.map do |choice|
|
|
91
|
-
Generation.new(
|
|
92
|
-
text: choice.dig("message", "content") || choice["text"],
|
|
93
|
-
generation_info: {
|
|
94
|
-
finish_reason: choice.fetch("finish_reason", nil),
|
|
95
|
-
logprobs: choice.fetch("logprobs", nil)
|
|
96
|
-
}
|
|
97
|
-
)
|
|
98
|
-
end
|
|
99
|
-
end
|
|
100
|
-
|
|
101
86
|
# make sure we got a valid response
|
|
102
87
|
# @param response [Hash] The response to check.
|
|
103
88
|
# @param must_haves [Array<String>] The keys that must be in the response. Defaults to %w[choices].
|
|
104
89
|
# @raise [KeyError] if there is an issue with the access token.
|
|
105
90
|
# @raise [ValueError] if the response is not valid.
|
|
106
91
|
def check_response(response, must_haves: %w[choices])
|
|
107
|
-
if response['error']
|
|
92
|
+
if response['error'].is_a?(Hash)
|
|
108
93
|
code = response.dig('error', 'code')
|
|
109
94
|
msg = response.dig('error', 'message') || 'unknown error'
|
|
110
|
-
raise KeyError, "
|
|
95
|
+
raise KeyError, "GROQ_API_TOKEN not valid" if code == 'invalid_api_key'
|
|
111
96
|
|
|
112
97
|
raise ValueError, "Groq error: #{msg}"
|
|
113
98
|
end
|
|
@@ -117,58 +102,20 @@ module Boxcars
|
|
|
117
102
|
end
|
|
118
103
|
end
|
|
119
104
|
|
|
120
|
-
#
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
124
|
-
# @return [EngineResult] The full engine output.
|
|
125
|
-
def generate(prompts:, stop: nil)
|
|
126
|
-
params = {}
|
|
127
|
-
params[:stop] = stop if stop
|
|
128
|
-
choices = []
|
|
129
|
-
token_usage = {}
|
|
130
|
-
# Get the token usage from the response.
|
|
131
|
-
# Includes prompt, completion, and total tokens used.
|
|
132
|
-
inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
|
|
133
|
-
prompts.each_slice(batch_size) do |sub_prompts|
|
|
134
|
-
sub_prompts.each do |sprompts, inputs|
|
|
135
|
-
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
136
|
-
check_response(response)
|
|
137
|
-
choices.concat(response["choices"])
|
|
138
|
-
usage_keys = inkeys & response["usage"].keys
|
|
139
|
-
usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
|
|
140
|
-
end
|
|
141
|
-
end
|
|
142
|
-
|
|
143
|
-
n = params.fetch(:n, 1)
|
|
144
|
-
generations = []
|
|
145
|
-
prompts.each_with_index do |_prompt, i|
|
|
146
|
-
sub_choices = choices[i * n, (i + 1) * n]
|
|
147
|
-
generations.push(generation_info(sub_choices))
|
|
148
|
-
end
|
|
149
|
-
EngineResult.new(generations: generations, engine_output: { token_usage: token_usage })
|
|
105
|
+
# the engine type
|
|
106
|
+
def engine_type
|
|
107
|
+
"groq"
|
|
150
108
|
end
|
|
151
|
-
# rubocop:enable Metrics/AbcSize
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
# the engine type
|
|
155
|
-
def engine_type
|
|
156
|
-
"groq"
|
|
157
|
-
end
|
|
158
109
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
# Calculate the maximum number of tokens possible to generate for a prompt.
|
|
165
|
-
# @param prompt_text [String] The prompt text to use.
|
|
166
|
-
# @return [Integer] the number of tokens possible to generate.
|
|
167
|
-
def max_tokens_for_prompt(prompt_text)
|
|
168
|
-
num_tokens = get_num_tokens(prompt_text)
|
|
110
|
+
# Calculate the maximum number of tokens possible to generate for a prompt.
|
|
111
|
+
# @param prompt_text [String] The prompt text to use.
|
|
112
|
+
# @return [Integer] the number of tokens possible to generate.
|
|
113
|
+
def max_tokens_for_prompt(prompt_text)
|
|
114
|
+
num_tokens = get_num_tokens(prompt_text)
|
|
169
115
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
116
|
+
# get max context size for model by name
|
|
117
|
+
max_size = 8096
|
|
118
|
+
max_size - num_tokens
|
|
119
|
+
end
|
|
173
120
|
end
|
|
174
121
|
end
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Boxcars is a framework for running a series of tools to get an answer to a question.
|
|
4
|
+
module Boxcars
|
|
5
|
+
# A engine that uses local GPT4All API.
|
|
6
|
+
class Ollama < Engine
|
|
7
|
+
attr_reader :prompts, :model_kwargs, :batch_size, :ollama_params
|
|
8
|
+
|
|
9
|
+
# The default parameters to use when asking the engine.
|
|
10
|
+
DEFAULT_PARAMS = {
|
|
11
|
+
model: "llama3",
|
|
12
|
+
temperature: 0.1,
|
|
13
|
+
max_tokens: 4096
|
|
14
|
+
}.freeze
|
|
15
|
+
|
|
16
|
+
# the default name of the engine
|
|
17
|
+
DEFAULT_NAME = "Ollama engine"
|
|
18
|
+
# the default description of the engine
|
|
19
|
+
DEFAULT_DESCRIPTION = "useful for when you need to use local AI to answer questions. " \
|
|
20
|
+
"You should ask targeted questions"
|
|
21
|
+
|
|
22
|
+
# A engine is a container for a single tool to run.
|
|
23
|
+
# @param name [String] The name of the engine. Defaults to "OpenAI engine".
|
|
24
|
+
# @param description [String] A description of the engine. Defaults to:
|
|
25
|
+
# useful for when you need to use AI to answer questions. You should ask targeted questions".
|
|
26
|
+
# @param prompts [Array<String>] The prompts to use when asking the engine. Defaults to [].
|
|
27
|
+
# @param batch_size [Integer] The number of prompts to send to the engine at once. Defaults to 2.
|
|
28
|
+
def initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, prompts: [], batch_size: 2, **kwargs)
|
|
29
|
+
@ollama_params = DEFAULT_PARAMS.merge(kwargs)
|
|
30
|
+
@prompts = prompts
|
|
31
|
+
@batch_size = batch_size
|
|
32
|
+
super(description: description, name: name)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Get the OpenAI API client
|
|
36
|
+
# @param groq_api_key [String] The access token to use when asking the engine.
|
|
37
|
+
# Defaults to Boxcars.configuration.groq_api_key
|
|
38
|
+
# @return [OpenAI::Client] The OpenAI API gem client.
|
|
39
|
+
def self.open_ai_client
|
|
40
|
+
::OpenAI::Client.new(uri_base: "http://localhost:11434")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def conversation_model?(_model)
|
|
44
|
+
true
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Get an answer from the engine.
|
|
48
|
+
# @param prompt [String] The prompt to use when asking the engine.
|
|
49
|
+
# @param groq_api_key [String] The access token to use when asking the engine.
|
|
50
|
+
# Defaults to Boxcars.configuration.groq_api_key.
|
|
51
|
+
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
|
52
|
+
def client(prompt:, inputs: {}, **kwargs)
|
|
53
|
+
clnt = Ollama.open_ai_client
|
|
54
|
+
params = ollama_params.merge(kwargs)
|
|
55
|
+
prompt = prompt.first if prompt.is_a?(Array)
|
|
56
|
+
params = prompt.as_messages(inputs).merge(params)
|
|
57
|
+
if Boxcars.configuration.log_prompts
|
|
58
|
+
Boxcars.debug(params[:messages].last(2).map { |p| ">>>>>> Role: #{p[:role]} <<<<<<\n#{p[:content]}" }.join("\n"), :cyan)
|
|
59
|
+
end
|
|
60
|
+
ans = clnt.chat(parameters: params)
|
|
61
|
+
ans['choices'].pluck('message').pluck('content').join("\n")
|
|
62
|
+
rescue => e
|
|
63
|
+
Boxcars.error(e, :red)
|
|
64
|
+
raise
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# get an answer from the engine for a question.
|
|
68
|
+
# @param question [String] The question to ask the engine.
|
|
69
|
+
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
|
70
|
+
def run(question, **kwargs)
|
|
71
|
+
prompt = Prompt.new(template: question)
|
|
72
|
+
answer = client(prompt: prompt, **kwargs)
|
|
73
|
+
raise Error, "Ollama: No response from API" unless answer
|
|
74
|
+
|
|
75
|
+
# raise Error, "Ollama: #{response['error']}" if response["error"]
|
|
76
|
+
Boxcars.debug("Answer: #{answer}", :cyan)
|
|
77
|
+
answer
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
@@ -88,21 +88,6 @@ module Boxcars
|
|
|
88
88
|
open_ai_params
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
-
# Get generation informaton
|
|
92
|
-
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
93
|
-
# @return [Array<Generation>] The generation information.
|
|
94
|
-
def generation_info(sub_choices)
|
|
95
|
-
sub_choices.map do |choice|
|
|
96
|
-
Generation.new(
|
|
97
|
-
text: choice.dig("message", "content") || choice["text"],
|
|
98
|
-
generation_info: {
|
|
99
|
-
finish_reason: choice.fetch("finish_reason", nil),
|
|
100
|
-
logprobs: choice.fetch("logprobs", nil)
|
|
101
|
-
}
|
|
102
|
-
)
|
|
103
|
-
end
|
|
104
|
-
end
|
|
105
|
-
|
|
106
91
|
# make sure we got a valid response
|
|
107
92
|
# @param response [Hash] The response to check.
|
|
108
93
|
# @param must_haves [Array<String>] The keys that must be in the response. Defaults to %w[choices].
|
|
@@ -121,39 +106,6 @@ module Boxcars
|
|
|
121
106
|
raise ValueError, "Expecting key #{key} in response" unless response.key?(key)
|
|
122
107
|
end
|
|
123
108
|
end
|
|
124
|
-
|
|
125
|
-
# Call out to OpenAI's endpoint with k unique prompts.
|
|
126
|
-
# @param prompts [Array<String>] The prompts to pass into the model.
|
|
127
|
-
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
|
128
|
-
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
129
|
-
# @return [EngineResult] The full engine output.
|
|
130
|
-
def generate(prompts:, stop: nil)
|
|
131
|
-
params = {}
|
|
132
|
-
params[:stop] = stop if stop
|
|
133
|
-
choices = []
|
|
134
|
-
token_usage = {}
|
|
135
|
-
# Get the token usage from the response.
|
|
136
|
-
# Includes prompt, completion, and total tokens used.
|
|
137
|
-
inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
|
|
138
|
-
prompts.each_slice(batch_size) do |sub_prompts|
|
|
139
|
-
sub_prompts.each do |sprompts, inputs|
|
|
140
|
-
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
141
|
-
check_response(response)
|
|
142
|
-
choices.concat(response["choices"])
|
|
143
|
-
usage_keys = inkeys & response["usage"].keys
|
|
144
|
-
usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
|
|
145
|
-
end
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
n = params.fetch(:n, 1)
|
|
149
|
-
generations = []
|
|
150
|
-
prompts.each_with_index do |_prompt, i|
|
|
151
|
-
sub_choices = choices[i * n, (i + 1) * n]
|
|
152
|
-
generations.push(generation_info(sub_choices))
|
|
153
|
-
end
|
|
154
|
-
EngineResult.new(generations: generations, engine_output: { token_usage: token_usage })
|
|
155
|
-
end
|
|
156
|
-
# rubocop:enable Metrics/AbcSize
|
|
157
109
|
end
|
|
158
110
|
|
|
159
111
|
# the engine type
|
|
@@ -161,11 +113,6 @@ module Boxcars
|
|
|
161
113
|
"openai"
|
|
162
114
|
end
|
|
163
115
|
|
|
164
|
-
# calculate the number of tokens used
|
|
165
|
-
def get_num_tokens(text:)
|
|
166
|
-
text.split.length # TODO: hook up to token counting gem
|
|
167
|
-
end
|
|
168
|
-
|
|
169
116
|
# lookup the context size for a model by name
|
|
170
117
|
# @param modelname [String] The name of the model to lookup.
|
|
171
118
|
def modelname_to_contextsize(modelname)
|
|
@@ -8,9 +8,8 @@ module Boxcars
|
|
|
8
8
|
|
|
9
9
|
# The default parameters to use when asking the engine.
|
|
10
10
|
DEFAULT_PER_PARAMS = {
|
|
11
|
-
model: "llama-
|
|
12
|
-
temperature: 0.1
|
|
13
|
-
max_tokens: 3200
|
|
11
|
+
model: "'llama-3-sonar-large-32k-online'",
|
|
12
|
+
temperature: 0.1
|
|
14
13
|
}.freeze
|
|
15
14
|
|
|
16
15
|
# the default name of the engine
|
|
@@ -32,28 +31,26 @@ module Boxcars
|
|
|
32
31
|
super(description: description, name: name)
|
|
33
32
|
end
|
|
34
33
|
|
|
35
|
-
def conversation_model?(
|
|
36
|
-
|
|
34
|
+
def conversation_model?(_model)
|
|
35
|
+
true
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
def chat(parameters:)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
response = http.request(request)
|
|
56
|
-
JSON.parse(response.read_body)
|
|
39
|
+
conn = Faraday.new(url: "https://api.perplexity.ai/chat/completions") do |faraday|
|
|
40
|
+
faraday.request :json
|
|
41
|
+
faraday.response :json
|
|
42
|
+
faraday.response :raise_error
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
response = conn.post do |req|
|
|
46
|
+
req.headers['Authorization'] = "Bearer #{ENV.fetch('PERPLEXITY_API_KEY')}"
|
|
47
|
+
req.body = {
|
|
48
|
+
model: parameters[:model],
|
|
49
|
+
messages: parameters[:messages]
|
|
50
|
+
}
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
response.body
|
|
57
54
|
end
|
|
58
55
|
|
|
59
56
|
# Get an answer from the engine.
|
|
@@ -64,7 +61,6 @@ module Boxcars
|
|
|
64
61
|
def client(prompt:, inputs: {}, **kwargs)
|
|
65
62
|
prompt = prompt.first if prompt.is_a?(Array)
|
|
66
63
|
params = prompt.as_messages(inputs).merge(default_params).merge(kwargs)
|
|
67
|
-
params[:model] ||= "llama-2-70b-chat"
|
|
68
64
|
if Boxcars.configuration.log_prompts
|
|
69
65
|
Boxcars.debug(params[:messages].last(2).map { |p| ">>>>>> Role: #{p[:role]} <<<<<<\n#{p[:content]}" }.join("\n"), :cyan)
|
|
70
66
|
end
|
|
@@ -90,21 +86,6 @@ module Boxcars
|
|
|
90
86
|
perplexity_params
|
|
91
87
|
end
|
|
92
88
|
|
|
93
|
-
# Get generation informaton
|
|
94
|
-
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
95
|
-
# @return [Array<Generation>] The generation information.
|
|
96
|
-
def generation_info(sub_choices)
|
|
97
|
-
sub_choices.map do |choice|
|
|
98
|
-
Generation.new(
|
|
99
|
-
text: choice.dig("message", "content") || choice["text"],
|
|
100
|
-
generation_info: {
|
|
101
|
-
finish_reason: choice.fetch("finish_reason", nil),
|
|
102
|
-
logprobs: choice.fetch("logprobs", nil)
|
|
103
|
-
}
|
|
104
|
-
)
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
89
|
# make sure we got a valid response
|
|
109
90
|
# @param response [Hash] The response to check.
|
|
110
91
|
# @param must_haves [Array<String>] The keys that must be in the response. Defaults to %w[choices].
|
|
@@ -123,39 +104,6 @@ module Boxcars
|
|
|
123
104
|
raise ValueError, "Expecting key #{key} in response" unless response.key?(key)
|
|
124
105
|
end
|
|
125
106
|
end
|
|
126
|
-
|
|
127
|
-
# Call out to OpenAI's endpoint with k unique prompts.
|
|
128
|
-
# @param prompts [Array<String>] The prompts to pass into the model.
|
|
129
|
-
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
|
130
|
-
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
131
|
-
# @return [EngineResult] The full engine output.
|
|
132
|
-
def generate(prompts:, stop: nil)
|
|
133
|
-
params = {}
|
|
134
|
-
params[:stop] = stop if stop
|
|
135
|
-
choices = []
|
|
136
|
-
token_usage = {}
|
|
137
|
-
# Get the token usage from the response.
|
|
138
|
-
# Includes prompt, completion, and total tokens used.
|
|
139
|
-
inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
|
|
140
|
-
prompts.each_slice(batch_size) do |sub_prompts|
|
|
141
|
-
sub_prompts.each do |sprompts, inputs|
|
|
142
|
-
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
143
|
-
check_response(response)
|
|
144
|
-
choices.concat(response["choices"])
|
|
145
|
-
usage_keys = inkeys & response["usage"].keys
|
|
146
|
-
usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
|
|
147
|
-
end
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
n = params.fetch(:n, 1)
|
|
151
|
-
generations = []
|
|
152
|
-
prompts.each_with_index do |_prompt, i|
|
|
153
|
-
sub_choices = choices[i * n, (i + 1) * n]
|
|
154
|
-
generations.push(generation_info(sub_choices))
|
|
155
|
-
end
|
|
156
|
-
EngineResult.new(generations: generations, engine_output: { token_usage: token_usage })
|
|
157
|
-
end
|
|
158
|
-
# rubocop:enable Metrics/AbcSize
|
|
159
107
|
end
|
|
160
108
|
|
|
161
109
|
# the engine type
|
|
@@ -168,29 +116,10 @@ module Boxcars
|
|
|
168
116
|
text.split.length # TODO: hook up to token counting gem
|
|
169
117
|
end
|
|
170
118
|
|
|
171
|
-
# lookup the context size for a model by name
|
|
172
|
-
# @param modelname [String] The name of the model to lookup.
|
|
173
|
-
def modelname_to_contextsize(modelname)
|
|
174
|
-
model_lookup = {
|
|
175
|
-
'text-davinci-003': 4097,
|
|
176
|
-
'text-curie-001': 2048,
|
|
177
|
-
'text-babbage-001': 2048,
|
|
178
|
-
'text-ada-001': 2048,
|
|
179
|
-
'code-davinci-002': 8000,
|
|
180
|
-
'code-cushman-001': 2048,
|
|
181
|
-
'gpt-3.5-turbo-1': 4096
|
|
182
|
-
}.freeze
|
|
183
|
-
model_lookup[modelname] || 4097
|
|
184
|
-
end
|
|
185
|
-
|
|
186
119
|
# Calculate the maximum number of tokens possible to generate for a prompt.
|
|
187
120
|
# @param prompt_text [String] The prompt text to use.
|
|
188
121
|
# @return [Integer] the number of tokens possible to generate.
|
|
189
|
-
def max_tokens_for_prompt(
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
# get max context size for model by name
|
|
193
|
-
max_size = modelname_to_contextsize(model_name)
|
|
194
|
-
max_size - num_tokens
|
|
122
|
+
def max_tokens_for_prompt(_prompt_text)
|
|
123
|
+
8096
|
|
195
124
|
end
|
|
196
125
|
end
|
data/lib/boxcars/engine.rb
CHANGED
|
@@ -16,6 +16,59 @@ module Boxcars
|
|
|
16
16
|
def run(question)
|
|
17
17
|
raise NotImplementedError
|
|
18
18
|
end
|
|
19
|
+
|
|
20
|
+
# calculate the number of tokens used
|
|
21
|
+
def get_num_tokens(text:)
|
|
22
|
+
text.split.length # TODO: hook up to token counting gem
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Get generation informaton
|
|
26
|
+
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
27
|
+
# @return [Array<Generation>] The generation information.
|
|
28
|
+
def generation_info(sub_choices)
|
|
29
|
+
sub_choices.map do |choice|
|
|
30
|
+
Generation.new(
|
|
31
|
+
text: choice.dig("message", "content") || choice["text"],
|
|
32
|
+
generation_info: {
|
|
33
|
+
finish_reason: choice.fetch("finish_reason", nil),
|
|
34
|
+
logprobs: choice.fetch("logprobs", nil)
|
|
35
|
+
}
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Call out to OpenAI's endpoint with k unique prompts.
|
|
41
|
+
# @param prompts [Array<String>] The prompts to pass into the model.
|
|
42
|
+
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
|
43
|
+
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
44
|
+
# @return [EngineResult] The full engine output.
|
|
45
|
+
def generate(prompts:, stop: nil)
|
|
46
|
+
params = {}
|
|
47
|
+
params[:stop] = stop if stop
|
|
48
|
+
choices = []
|
|
49
|
+
token_usage = {}
|
|
50
|
+
# Get the token usage from the response.
|
|
51
|
+
# Includes prompt, completion, and total tokens used.
|
|
52
|
+
inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
|
|
53
|
+
prompts.each_slice(batch_size) do |sub_prompts|
|
|
54
|
+
sub_prompts.each do |sprompts, inputs|
|
|
55
|
+
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
56
|
+
check_response(response)
|
|
57
|
+
choices.concat(response["choices"])
|
|
58
|
+
usage_keys = inkeys & response["usage"].keys
|
|
59
|
+
usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
n = params.fetch(:n, 1)
|
|
64
|
+
generations = []
|
|
65
|
+
prompts.each_with_index do |_prompt, i|
|
|
66
|
+
sub_choices = choices[i * n, (i + 1) * n]
|
|
67
|
+
generations.push(generation_info(sub_choices))
|
|
68
|
+
end
|
|
69
|
+
EngineResult.new(generations: generations, engine_output: { token_usage: token_usage })
|
|
70
|
+
end
|
|
71
|
+
# rubocop:enable Metrics/AbcSize
|
|
19
72
|
end
|
|
20
73
|
end
|
|
21
74
|
|
|
@@ -23,6 +76,7 @@ require "boxcars/engine/engine_result"
|
|
|
23
76
|
require "boxcars/engine/anthropic"
|
|
24
77
|
require "boxcars/engine/cohere"
|
|
25
78
|
require "boxcars/engine/groq"
|
|
79
|
+
require "boxcars/engine/ollama"
|
|
26
80
|
require "boxcars/engine/openai"
|
|
27
81
|
require "boxcars/engine/perplexityai"
|
|
28
82
|
require "boxcars/engine/gpt4all_eng"
|
data/lib/boxcars/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: boxcars
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Francis Sullivan
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2024-07-
|
|
12
|
+
date: 2024-07-27 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: anthropic
|
|
@@ -161,6 +161,7 @@ files:
|
|
|
161
161
|
- lib/boxcars/engine/engine_result.rb
|
|
162
162
|
- lib/boxcars/engine/gpt4all_eng.rb
|
|
163
163
|
- lib/boxcars/engine/groq.rb
|
|
164
|
+
- lib/boxcars/engine/ollama.rb
|
|
164
165
|
- lib/boxcars/engine/openai.rb
|
|
165
166
|
- lib/boxcars/engine/perplexityai.rb
|
|
166
167
|
- lib/boxcars/generation.rb
|
|
@@ -218,7 +219,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
218
219
|
- !ruby/object:Gem::Version
|
|
219
220
|
version: '0'
|
|
220
221
|
requirements: []
|
|
221
|
-
rubygems_version: 3.
|
|
222
|
+
rubygems_version: 3.5.11
|
|
222
223
|
signing_key:
|
|
223
224
|
specification_version: 4
|
|
224
225
|
summary: Boxcars is a gem that enables you to create new systems with AI composability.
|