boxcars 0.2.1 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/Gemfile.lock +1 -1
- data/lib/boxcars/boxcar/active_record.rb +69 -37
- data/lib/boxcars/boxcar/calculator.rb +29 -40
- data/lib/boxcars/boxcar/engine_boxcar.rb +59 -46
- data/lib/boxcars/boxcar/sql.rb +26 -27
- data/lib/boxcars/boxcar.rb +16 -0
- data/lib/boxcars/conversation.rb +98 -0
- data/lib/boxcars/conversation_prompt.rb +40 -0
- data/lib/boxcars/engine/openai.rb +23 -15
- data/lib/boxcars/prompt.rb +25 -38
- data/lib/boxcars/train/zero_shot.rb +59 -49
- data/lib/boxcars/train.rb +21 -20
- data/lib/boxcars/version.rb +1 -1
- data/lib/boxcars.rb +9 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ea9c5df0584f588d618c22fc44f8bba2638c0946ba32cea8487429ad6df694f5
|
4
|
+
data.tar.gz: a7cd6609da5d63c1b41763fc7c6e8b4e1c8df41992fd133d0bf359b28d50dd0c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aac8bd34e6ddca629d26ff0cb892e46de30e1f20db188b56477422e5526e03c80a22e1b06ab7d8263e8db067db6ec1966f34450ab63e98cea3e857aa7eb27286
|
7
|
+
data.tar.gz: 10ce3be58a020a5de80c27fdf46cbd7811afc45ac28e0712479b113eb6a34ca6a2f7043711659bb4b0a152e0981ded5a8f76fe0704f1a89751b3c8ec0a843ea8
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,40 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## [Unreleased](https://github.com/BoxcarsAI/boxcars/tree/HEAD)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.2...HEAD)
|
6
|
+
|
7
|
+
**Merged pull requests:**
|
8
|
+
|
9
|
+
- better error handling and retry for Active Record Boxcar [\#39](https://github.com/BoxcarsAI/boxcars/pull/39) ([francis](https://github.com/francis))
|
10
|
+
|
11
|
+
## [v0.2.2](https://github.com/BoxcarsAI/boxcars/tree/v0.2.2) (2023-03-16)
|
12
|
+
|
13
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.1...v0.2.2)
|
14
|
+
|
15
|
+
**Implemented enhancements:**
|
16
|
+
|
17
|
+
- return a structure from boxcars instead of just a string [\#31](https://github.com/BoxcarsAI/boxcars/issues/31)
|
18
|
+
- modify SQL boxcar to take a list of Tables. Handy if you have a large database with many tables. [\#19](https://github.com/BoxcarsAI/boxcars/issues/19)
|
19
|
+
|
20
|
+
**Closed issues:**
|
21
|
+
|
22
|
+
- make a new type of prompt that uses conversations [\#38](https://github.com/BoxcarsAI/boxcars/issues/38)
|
23
|
+
- Add test coverage [\#8](https://github.com/BoxcarsAI/boxcars/issues/8)
|
24
|
+
|
25
|
+
## [v0.2.1](https://github.com/BoxcarsAI/boxcars/tree/v0.2.1) (2023-03-08)
|
26
|
+
|
27
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.0...v0.2.1)
|
28
|
+
|
29
|
+
**Implemented enhancements:**
|
30
|
+
|
31
|
+
- add the ability to use the ChatGPT API - a ChatGPT Boxcar::Engine [\#32](https://github.com/BoxcarsAI/boxcars/issues/32)
|
32
|
+
- Prompt simplification [\#29](https://github.com/BoxcarsAI/boxcars/issues/29)
|
33
|
+
|
34
|
+
**Merged pull requests:**
|
35
|
+
|
36
|
+
- use structured results internally and bring SQL up to parity with ActiveRecord boxcar. [\#36](https://github.com/BoxcarsAI/boxcars/pull/36) ([francis](https://github.com/francis))
|
37
|
+
|
3
38
|
## [v0.2.0](https://github.com/BoxcarsAI/boxcars/tree/v0.2.0) (2023-03-07)
|
4
39
|
|
5
40
|
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.1.8...v0.2.0)
|
data/Gemfile.lock
CHANGED
@@ -71,13 +71,21 @@ module Boxcars
|
|
71
71
|
|
72
72
|
# to be safe, we wrap the code in a transaction and rollback
|
73
73
|
def rollback_after_running
|
74
|
-
|
74
|
+
result = nil
|
75
|
+
runtime_exception = nil
|
75
76
|
::ActiveRecord::Base.transaction do
|
76
|
-
|
77
|
+
begin
|
78
|
+
result = yield
|
79
|
+
rescue ::NameError, ::Error => e
|
80
|
+
Boxcars.error("Error while running code: #{e.message[0..60]} ...", :red)
|
81
|
+
runtime_exception = e
|
82
|
+
end
|
77
83
|
ensure
|
78
84
|
raise ::ActiveRecord::Rollback
|
79
85
|
end
|
80
|
-
|
86
|
+
raise runtime_exception if runtime_exception
|
87
|
+
|
88
|
+
result
|
81
89
|
end
|
82
90
|
|
83
91
|
# check for dangerous code that is outside of ActiveRecord
|
@@ -122,8 +130,11 @@ module Boxcars
|
|
122
130
|
return true unless changes&.positive?
|
123
131
|
|
124
132
|
Boxcars.debug "#{name}(Pending Changes): #{changes}", :yellow
|
125
|
-
|
126
|
-
|
133
|
+
if read_only?
|
134
|
+
change_str = "#{changes} change#{'s' if changes.to_i > 1}"
|
135
|
+
Boxcars.error("Can not run code that makes #{change_str} in read-only mode", :red)
|
136
|
+
return false
|
137
|
+
end
|
127
138
|
|
128
139
|
return approval_callback.call(changes, code) if approval_callback.is_a?(Proc)
|
129
140
|
|
@@ -131,6 +142,7 @@ module Boxcars
|
|
131
142
|
end
|
132
143
|
|
133
144
|
def run_active_record_code(code)
|
145
|
+
code = ::Regexp.last_match(1) if code =~ /`(.+)`/
|
134
146
|
Boxcars.debug code, :yellow
|
135
147
|
if read_only?
|
136
148
|
rollback_after_running do
|
@@ -150,17 +162,32 @@ module Boxcars
|
|
150
162
|
output
|
151
163
|
end
|
152
164
|
|
165
|
+
def error_message(err, stage)
|
166
|
+
msg = err.message
|
167
|
+
msg = ::Regexp.last_match(1) if msg =~ /^(.+)' for #<Boxcars::ActiveRecord/
|
168
|
+
"#{stage} Error: #{msg} - please fix \"#{stage}:\" to not have this error"
|
169
|
+
end
|
170
|
+
|
153
171
|
def get_active_record_answer(text)
|
154
|
-
|
155
|
-
|
172
|
+
changes_code = extract_code text.split('ARCode:').first.split('ARChanges:').last.strip if text =~ /^ARChanges:/
|
173
|
+
code = extract_code text.split('ARCode:').last.strip
|
156
174
|
return Result.new(status: :ok, explanation: "code to run", code: code, changes_code: changes_code) if code_only?
|
157
175
|
|
158
|
-
|
176
|
+
have_approval = false
|
177
|
+
begin
|
178
|
+
have_approval = approved?(changes_code, code)
|
179
|
+
rescue NameError, ::Error => e
|
180
|
+
return Result.new(status: :error, explanation: error_message(e, "ARChanges"), changes_code: changes_code)
|
181
|
+
end
|
182
|
+
|
183
|
+
raise SecurityError, "Permission to run code that makes changes denied" unless have_approval
|
159
184
|
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
185
|
+
begin
|
186
|
+
output = clean_up_output(run_active_record_code(code))
|
187
|
+
Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
|
188
|
+
rescue NameError, ::Error => e
|
189
|
+
Result.new(status: :error, answer: nil, explanation: error_message(e, "ARCode"), code: code)
|
190
|
+
end
|
164
191
|
end
|
165
192
|
|
166
193
|
def get_answer(text)
|
@@ -170,37 +197,42 @@ module Boxcars
|
|
170
197
|
when /^Answer:/
|
171
198
|
Result.from_text(text)
|
172
199
|
else
|
173
|
-
Result.from_error("
|
200
|
+
Result.from_error("Error: Your answer wasn't formatted properly - try again. I expected your answer to " \
|
201
|
+
"start with \"ARChanges:\" or \"ARCode:\"")
|
174
202
|
end
|
175
203
|
end
|
176
204
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
Question: %<question>s
|
198
|
-
|
205
|
+
CTEMPLATE = [
|
206
|
+
syst("You are a Ruby on Rails Active Record code generator"),
|
207
|
+
syst("Given an input question, first create a syntactically correct Rails Active Record code to run, ",
|
208
|
+
"then look at the results of the code and return the answer. Unless the user specifies ",
|
209
|
+
"in her question a specific number of examples she wishes to obtain, limit your code ",
|
210
|
+
"to at most %<top_k>s results.\n",
|
211
|
+
"Never query for all the columns from a specific model, ",
|
212
|
+
"only ask for the relevant attributes given the question.\n",
|
213
|
+
"Also, pay attention to which attribute is in which model.\n\n",
|
214
|
+
"Use the following format:\n",
|
215
|
+
"Question: ${{Question here}}\n",
|
216
|
+
"ARChanges: ${{Active Record code to compute the number of records going to change}} - ",
|
217
|
+
"Only add this line if the ARCode on the next line will make data changes.\n",
|
218
|
+
"ARCode: ${{Active Record code to run}} - make sure you use valid code\n",
|
219
|
+
"Answer: ${{Final answer here}}\n\n",
|
220
|
+
"Only use the following Active Record models: %<model_info>s\n",
|
221
|
+
"Pay attention to use only the attribute names that you can see in the model description.\n",
|
222
|
+
"Do not make up variable or attribute names, and do not share variables between the code in ARChanges and ARCode\n",
|
223
|
+
"Be careful to not query for attributes that do not exist, and to use the format specified above.\n"
|
224
|
+
),
|
225
|
+
user("Question: %<question>s")
|
226
|
+
].freeze
|
199
227
|
|
200
228
|
# The prompt to use for the engine.
|
201
229
|
def my_prompt
|
202
|
-
@
|
203
|
-
|
230
|
+
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
231
|
+
@my_prompt ||= ConversationPrompt.new(
|
232
|
+
conversation: @conversation,
|
233
|
+
input_variables: [:question],
|
234
|
+
other_inputs: [:top_k],
|
235
|
+
output_variables: [:answer])
|
204
236
|
end
|
205
237
|
end
|
206
238
|
end
|
@@ -21,7 +21,8 @@ module Boxcars
|
|
21
21
|
private
|
22
22
|
|
23
23
|
def get_embedded_ruby_answer(text)
|
24
|
-
code = text
|
24
|
+
code = text.split("```ruby\n").last.split("```").first.strip
|
25
|
+
# code = text[8..-4].split("```").first.strip
|
25
26
|
ruby_executor = Boxcars::RubyREPL.new
|
26
27
|
ruby_executor.call(code: code)
|
27
28
|
end
|
@@ -33,52 +34,40 @@ module Boxcars
|
|
33
34
|
when /^Answer:/
|
34
35
|
Result.from_text(text)
|
35
36
|
else
|
36
|
-
Result.new(status: :error,
|
37
|
+
Result.new(status: :error,
|
38
|
+
explanation: "Error: expecting your response to begin with '```ruby'. Try answering the question again.")
|
37
39
|
end
|
38
40
|
end
|
39
41
|
|
40
42
|
# our template
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
Begin.
|
62
|
-
|
63
|
-
Question: What is 37593 * 67?
|
64
|
-
```ruby
|
65
|
-
puts(37593 * 67)
|
66
|
-
```
|
67
|
-
```output
|
68
|
-
2518731
|
69
|
-
```
|
70
|
-
Answer: 2518731
|
71
|
-
|
72
|
-
Question: what is 2518731 + 0?
|
73
|
-
Answer: 2518731
|
74
|
-
|
75
|
-
Question: %<question>s
|
76
|
-
IPT
|
77
|
-
# rubocop:enable Style/RedundantHeredocDelimiterQuotes
|
43
|
+
CTEMPLATE = [
|
44
|
+
syst("You can do basic math, but for any hard calculations that a human could not do ",
|
45
|
+
"in their head, use the following approach instead. ",
|
46
|
+
"Return code written in the Ruby programming language that prints the results. ",
|
47
|
+
"If anyone gives you a hard math problem, just ",
|
48
|
+
"use the following format and we’ll take care of the rest:\n",
|
49
|
+
"${{Question with hard calculation.}}\n",
|
50
|
+
"reply only with the following format:\n",
|
51
|
+
"```ruby\n${{only Ruby code that prints the answer}}\n```\n",
|
52
|
+
"```output\n${{Output of your code}}\n```\n\n",
|
53
|
+
"Otherwise, you should use this simpler format:\n",
|
54
|
+
"${{Question without hard calculation}}\n",
|
55
|
+
"Answer: ${{Answer}}\n\n",
|
56
|
+
"Do not give an explanation of the answer and make sure your answer starts with either 'Answer:' or '```ruby'."),
|
57
|
+
syst("here is a hard example:\n", "the user asks: What is 37593 * 67?\n",
|
58
|
+
"your answer: ```ruby\nputs(37593 * 67)\n```\n```output\n2518731\n```\nAnswer: 2518731"),
|
59
|
+
syst("basic example:\n", "user asks: What is 2518731 + 0?\n", "you answer: Answer: 2518731"),
|
60
|
+
syst("Begin."),
|
61
|
+
user("%<question>s")
|
62
|
+
].freeze
|
78
63
|
|
79
64
|
# The prompt to use for the engine.
|
80
65
|
def my_prompt
|
81
|
-
@
|
66
|
+
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
67
|
+
@my_prompt ||= ConversationPrompt.new(
|
68
|
+
conversation: @conversation,
|
69
|
+
input_variables: [:question],
|
70
|
+
output_variables: [:answer])
|
82
71
|
end
|
83
72
|
end
|
84
73
|
end
|
@@ -8,15 +8,14 @@ module Boxcars
|
|
8
8
|
|
9
9
|
# A Boxcar is a container for a single tool to run.
|
10
10
|
# @param prompt [Boxcars::Prompt] The prompt to use for this boxcar with sane defaults.
|
11
|
-
# @param name [String] The name of the boxcar. Defaults to classname.
|
12
|
-
# @param description [String] A description of the boxcar.
|
13
11
|
# @param engine [Boxcars::Engine] The engine to user for this boxcar. Can be inherited from a train if nil.
|
14
|
-
|
12
|
+
# @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
|
13
|
+
def initialize(prompt:, engine: nil, **kwargs)
|
15
14
|
@prompt = prompt
|
16
15
|
@engine = engine || Boxcars.engine.new
|
17
|
-
@top_k = kwargs
|
18
|
-
@stop = kwargs
|
19
|
-
super(
|
16
|
+
@top_k = kwargs.delete(:top_k) || 5
|
17
|
+
@stop = kwargs.delete(:stop) || ["Answer:"]
|
18
|
+
super(**kwargs)
|
20
19
|
end
|
21
20
|
|
22
21
|
# input keys for the prompt
|
@@ -34,60 +33,41 @@ module Boxcars
|
|
34
33
|
prompt.output_variables
|
35
34
|
end
|
36
35
|
|
36
|
+
# the first output key
|
37
|
+
def output_key
|
38
|
+
output_keys.first
|
39
|
+
end
|
40
|
+
|
37
41
|
# generate a response from the engine
|
38
42
|
# @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
|
43
|
+
# @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
|
39
44
|
# @return [Boxcars::EngineResult] The result from the engine.
|
40
|
-
def generate(input_list:)
|
45
|
+
def generate(input_list:, current_conversation: nil)
|
41
46
|
stop = input_list[0][:stop]
|
42
|
-
|
43
|
-
input_list.
|
44
|
-
# prompt.missing_variables?(inputs)
|
45
|
-
new_prompt = prompt.format(**inputs)
|
46
|
-
Boxcars.debug("Prompt after formatting:\n#{new_prompt}", :cyan) if Boxcars.configuration.log_prompts
|
47
|
-
prompts.push(new_prompt)
|
48
|
-
end
|
47
|
+
the_prompt = current_conversation ? prompt.with_conversation(current_conversation) : prompt
|
48
|
+
prompts = input_list.map { |inputs| [the_prompt, inputs] }
|
49
49
|
engine.generate(prompts: prompts, stop: stop)
|
50
50
|
end
|
51
51
|
|
52
52
|
# apply a response from the engine
|
53
53
|
# @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
|
54
|
+
# @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
|
54
55
|
# @return [Hash] A hash of the output key and the output value.
|
55
|
-
def apply(input_list:)
|
56
|
-
response = generate(input_list: input_list)
|
56
|
+
def apply(input_list:, current_conversation: nil)
|
57
|
+
response = generate(input_list: input_list, current_conversation: current_conversation)
|
57
58
|
response.generations.to_h do |generation|
|
58
59
|
[output_keys.first, generation[0].text]
|
59
60
|
end
|
60
61
|
end
|
61
62
|
|
62
63
|
# predict a response from the engine
|
64
|
+
# @param current_conversation [Boxcars::Conversation] Optional ongoing conversation to use for the prompt.
|
63
65
|
# @param kwargs [Hash] A hash of input values to use for the prompt.
|
64
66
|
# @return [String] The output value.
|
65
|
-
def predict(**kwargs)
|
66
|
-
apply(input_list: [kwargs])[output_keys.first]
|
67
|
-
|
68
|
-
|
69
|
-
# predict a response from the engine and parse it
|
70
|
-
# @param kwargs [Hash] A hash of input values to use for the prompt.
|
71
|
-
# @return [String] The output value.
|
72
|
-
def predict_and_parse(**kwargs)
|
73
|
-
result = predict(**kwargs)
|
74
|
-
if prompt.output_parser
|
75
|
-
prompt.output_parser.parse(result)
|
76
|
-
else
|
77
|
-
result
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
# apply a response from the engine and parse it
|
82
|
-
# @param input_list [Array<Hash>] A list of hashes of input values to use for the prompt.
|
83
|
-
# @return [Array<String>] The output values.
|
84
|
-
def apply_and_parse(input_list:)
|
85
|
-
result = apply(input_list: input_list)
|
86
|
-
if prompt.output_parser
|
87
|
-
result.map { |r| prompt.output_parser.parse(r[output_keys.first]) }
|
88
|
-
else
|
89
|
-
result
|
90
|
-
end
|
67
|
+
def predict(current_conversation: nil, **kwargs)
|
68
|
+
prediction = apply(current_conversation: current_conversation, input_list: [kwargs])[output_keys.first]
|
69
|
+
Boxcars.debug(prediction, :white) if Boxcars.configuration.log_generated
|
70
|
+
prediction
|
91
71
|
end
|
92
72
|
|
93
73
|
# check that there is exactly one output key
|
@@ -102,10 +82,29 @@ module Boxcars
|
|
102
82
|
# @param inputs [Hash] The inputs to the boxcar.
|
103
83
|
# @return [Hash] The outputs from the boxcar.
|
104
84
|
def call(inputs:)
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
85
|
+
# if we get errors back, try predicting again giving the errors with the inputs
|
86
|
+
conversation = nil
|
87
|
+
answer = nil
|
88
|
+
4.times do
|
89
|
+
text = predict(current_conversation: conversation, **prediction_variables(inputs)).strip
|
90
|
+
answer = get_answer(text)
|
91
|
+
if answer.status == :error
|
92
|
+
Boxcars.debug "have error, trying again: #{answer.answer}", :red
|
93
|
+
conversation ||= Conversation.new
|
94
|
+
conversation.add_assistant(text)
|
95
|
+
conversation.add_user(answer.answer)
|
96
|
+
else
|
97
|
+
Boxcars.debug answer.to_json, :magenta
|
98
|
+
return { output_keys.first => answer }
|
99
|
+
end
|
100
|
+
end
|
101
|
+
Boxcars.error answer.to_json, :red
|
102
|
+
{ output_key => "Error: #{answer}" }
|
103
|
+
rescue Boxcars::ConfigurationError => e
|
104
|
+
raise e
|
105
|
+
rescue Boxcars::Error => e
|
106
|
+
Boxcars.error e.message, :red
|
107
|
+
{ output_key => "Error: #{e.message}" }
|
109
108
|
end
|
110
109
|
|
111
110
|
# @param inputs [Hash] The inputs to the boxcar.
|
@@ -124,5 +123,19 @@ module Boxcars
|
|
124
123
|
def prediction_variables(inputs)
|
125
124
|
prediction_input(inputs).merge(prediction_additional)
|
126
125
|
end
|
126
|
+
|
127
|
+
# remove backticks or triple backticks from the code
|
128
|
+
# @param code [String] The code to remove backticks from.
|
129
|
+
# @return [String] The code without backticks.
|
130
|
+
def extract_code(code)
|
131
|
+
case code
|
132
|
+
when /^```\w*/
|
133
|
+
code.split(/```\w*\n/).last.split('```').first.strip
|
134
|
+
when /^`(.+)`/
|
135
|
+
::Regexp.last_match(1)
|
136
|
+
else
|
137
|
+
code.gsub("`", "")
|
138
|
+
end
|
139
|
+
end
|
127
140
|
end
|
128
141
|
end
|
data/lib/boxcars/boxcar/sql.rb
CHANGED
@@ -20,6 +20,8 @@ module Boxcars
|
|
20
20
|
kwargs[:name] ||= "Database"
|
21
21
|
kwargs[:description] ||= format(SQLDESC, name: name)
|
22
22
|
kwargs[:prompt] ||= my_prompt
|
23
|
+
kwargs[:stop] ||= ["SQLResult:"]
|
24
|
+
|
23
25
|
super(**kwargs)
|
24
26
|
end
|
25
27
|
|
@@ -73,6 +75,7 @@ module Boxcars
|
|
73
75
|
|
74
76
|
def get_embedded_sql_answer(text)
|
75
77
|
code = text[/^SQLQuery: (.*)/, 1]
|
78
|
+
code = extract_code text.split('SQLQuery:').last.strip
|
76
79
|
Boxcars.debug code, :yellow
|
77
80
|
output = clean_up_output(connection.exec_query(code))
|
78
81
|
Result.new(status: :ok, answer: output, explanation: "Answer: #{output.to_json}", code: code)
|
@@ -87,41 +90,37 @@ module Boxcars
|
|
87
90
|
when /^Answer:/
|
88
91
|
Result.from_text(text)
|
89
92
|
else
|
90
|
-
Result.from_error("
|
93
|
+
Result.from_error("Your answer wasn't formatted properly - try again. I expected your answer to " \
|
94
|
+
"start with \"SQLQuery:\".")
|
91
95
|
end
|
92
96
|
end
|
93
97
|
|
94
|
-
|
95
|
-
Given an input question, first create a syntactically correct %<dialect>s SQL query to run,
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
Only use the following tables:
|
113
|
-
%<schema>s
|
114
|
-
|
115
|
-
Question: %<question>s
|
116
|
-
IPT
|
98
|
+
CTEMPLATE = [
|
99
|
+
syst("Given an input question, first create a syntactically correct %<dialect>s SQL query to run, ",
|
100
|
+
"then look at the results of the query and return the answer. Unless the user specifies ",
|
101
|
+
"in her question a specific number of examples he wishes to obtain, always limit your query ",
|
102
|
+
"to at most %<top_k>s results using a LIMIT clause. You can order the results by a relevant column ",
|
103
|
+
"to return the most interesting examples in the database.\n",
|
104
|
+
"Never query for all the columns from a specific table, only ask for the elevant columns given the question.\n",
|
105
|
+
"Pay attention to use only the column names that you can see in the schema description. Be careful to ",
|
106
|
+
"not query for columns that do not exist. Also, pay attention to which column is in which table.\n",
|
107
|
+
"Use the following format:\n",
|
108
|
+
"Question: 'Question here'\n",
|
109
|
+
"SQLQuery: 'SQL Query to run'\n",
|
110
|
+
"SQLResult: 'Result of the SQLQuery'\n",
|
111
|
+
"Answer: 'Final answer here'"),
|
112
|
+
syst("Only use the following tables:\n%<schema>s"),
|
113
|
+
user("Question: %<question>s")
|
114
|
+
].freeze
|
117
115
|
|
118
116
|
# The prompt to use for the engine.
|
119
117
|
def my_prompt
|
120
|
-
@
|
118
|
+
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
119
|
+
@my_prompt ||= ConversationPrompt.new(
|
120
|
+
conversation: @conversation,
|
121
121
|
input_variables: [:question],
|
122
122
|
other_inputs: [:top_k, :dialect, :table_info],
|
123
|
-
output_variables: [:answer]
|
124
|
-
template: TEMPLATE)
|
123
|
+
output_variables: [:answer])
|
125
124
|
end
|
126
125
|
end
|
127
126
|
end
|
data/lib/boxcars/boxcar.rb
CHANGED
@@ -78,6 +78,22 @@ module Boxcars
|
|
78
78
|
rv
|
79
79
|
end
|
80
80
|
|
81
|
+
# helpers for conversation prompt building
|
82
|
+
# assistant message
|
83
|
+
def self.assi(*strs)
|
84
|
+
[:assistant, strs.join]
|
85
|
+
end
|
86
|
+
|
87
|
+
# system message
|
88
|
+
def self.syst(*strs)
|
89
|
+
[:system, strs.join]
|
90
|
+
end
|
91
|
+
|
92
|
+
# user message
|
93
|
+
def self.user(*strs)
|
94
|
+
[:user, strs.join]
|
95
|
+
end
|
96
|
+
|
81
97
|
private
|
82
98
|
|
83
99
|
# Get an answer from the boxcar.
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Boxcars
|
4
|
+
# used to keep track of the conversation
|
5
|
+
class Conversation
|
6
|
+
attr_reader :lines, :show_roles
|
7
|
+
|
8
|
+
PEOPLE = %i[system user assistant].freeze
|
9
|
+
|
10
|
+
def initialize(lines: [], show_roles: false)
|
11
|
+
@lines = lines
|
12
|
+
check_lines(@lines)
|
13
|
+
@show_roles = show_roles
|
14
|
+
end
|
15
|
+
|
16
|
+
# check the lines
|
17
|
+
def check_lines(lines)
|
18
|
+
raise ArgumentError, "Lines must be an array" unless lines.is_a?(Array)
|
19
|
+
|
20
|
+
lines.each do |ln|
|
21
|
+
raise ArgumentError, "Conversation item must be a array" unless ln.is_a?(Array)
|
22
|
+
raise ArgumentError, "Conversation item must have 2 items, role and text" unless ln.size == 2
|
23
|
+
raise ArgumentError, "Conversation item must have a role #{ln} in (#{PEOPLE})" unless PEOPLE.include? ln[0]
|
24
|
+
raise ArgumentError, "Conversation value must be a string" unless ln[1].is_a?(String)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# @return [Array] The result as a convesation array
|
29
|
+
def to_a
|
30
|
+
lines
|
31
|
+
end
|
32
|
+
|
33
|
+
# @return [String] A conversation string
|
34
|
+
def to_s
|
35
|
+
lines.map { |ln| "#{ln[0]}: #{ln[1]}" }.join("\n")
|
36
|
+
end
|
37
|
+
|
38
|
+
# add assistant text to the conversation at the end
|
39
|
+
# @param text [String] The text to add
|
40
|
+
def add_assistant(text)
|
41
|
+
@lines << [:assistant, text]
|
42
|
+
end
|
43
|
+
|
44
|
+
# add user text to the conversation at the end
|
45
|
+
# @param text [String] The text to add
|
46
|
+
def add_user(text)
|
47
|
+
@lines << [:user, text]
|
48
|
+
end
|
49
|
+
|
50
|
+
# add system text to the conversation at the end
|
51
|
+
# @param text [String] The text to add
|
52
|
+
def add_system(text)
|
53
|
+
@lines << [:system, text]
|
54
|
+
end
|
55
|
+
|
56
|
+
# add multiple lines to the conversation
|
57
|
+
def add_lines(lines)
|
58
|
+
check_lines(lines)
|
59
|
+
@lines += lines
|
60
|
+
end
|
61
|
+
|
62
|
+
# add a conversation to the conversation
|
63
|
+
def add_conversation(conversation)
|
64
|
+
@lines += conversation.lines
|
65
|
+
end
|
66
|
+
|
67
|
+
# return just the messages for the conversation
|
68
|
+
def message_text
|
69
|
+
lines.map(&:last).join("\n")
|
70
|
+
end
|
71
|
+
|
72
|
+
# compute the prompt parameters with input substitutions (used for chatGPT)
|
73
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
74
|
+
# @return [Hash] The formatted prompt { messages: ...}
|
75
|
+
def as_messages(inputs = nil)
|
76
|
+
{ messages: lines.map { |ln| { role: ln.first, content: ln.last % inputs } } }
|
77
|
+
rescue ::KeyError => e
|
78
|
+
first_line = e.message.to_s.split("\n").first
|
79
|
+
Boxcars.error "Missing prompt input key: #{first_line}"
|
80
|
+
raise KeyError, "Prompt format error: #{first_line}"
|
81
|
+
end
|
82
|
+
|
83
|
+
# compute the prompt parameters with input substitutions
|
84
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
85
|
+
# @return [Hash] The formatted prompt { prompt: "..."}
|
86
|
+
def as_prompt(inputs = nil)
|
87
|
+
if show_roles
|
88
|
+
lines.map { |ln| format("#{ln.first}: #{ln.last}", inputs) }.join("\n\n")
|
89
|
+
else
|
90
|
+
lines.map { |ln| format(ln.last, inputs) }.join("\n\n")
|
91
|
+
end
|
92
|
+
rescue ::KeyError => e
|
93
|
+
first_line = e.message.to_s.split("\n").first
|
94
|
+
Boxcars.error "Missing prompt input key: #{first_line}"
|
95
|
+
raise KeyError, "Prompt format error: #{first_line}"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Boxcars
|
4
|
+
# used by Boxcars that have engine's to create a conversation prompt.
|
5
|
+
class ConversationPrompt < Prompt
|
6
|
+
attr_reader :conversation
|
7
|
+
|
8
|
+
# @param conversation [Boxcars::Conversation] The conversation to use for the prompt.
|
9
|
+
# @param input_variables [Array<Symbol>] The input vars to use for the prompt. Defaults to [:input]
|
10
|
+
# @param other_inputs [Array<Symbol>] The other input vars to use for the prompt. Defaults to []
|
11
|
+
# @param output_variables [Array<Symbol>] The output vars to use for the prompt. Defaults to [:output]
|
12
|
+
def initialize(conversation:, input_variables: nil, other_inputs: nil, output_variables: nil)
|
13
|
+
@conversation = conversation
|
14
|
+
super(template: template, input_variables: input_variables, other_inputs: other_inputs, output_variables: output_variables)
|
15
|
+
end
|
16
|
+
|
17
|
+
# prompt for chatGPT params
|
18
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
19
|
+
# @return [Hash] The formatted prompt.
|
20
|
+
def as_messages(inputs)
|
21
|
+
conversation.as_messages(inputs)
|
22
|
+
end
|
23
|
+
|
24
|
+
# prompt for non chatGPT params
|
25
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
26
|
+
# @return [Hash] The formatted prompt.
|
27
|
+
def as_prompt(inputs)
|
28
|
+
{ prompt: conversation.as_prompt(inputs) }
|
29
|
+
end
|
30
|
+
|
31
|
+
# tack on the ongoing conversation if present to the prompt
|
32
|
+
def with_conversation(conversation)
|
33
|
+
return self unless conversation
|
34
|
+
|
35
|
+
new_prompt = dup
|
36
|
+
new_prompt.conversation.add_conversation(conversation)
|
37
|
+
new_prompt
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -10,7 +10,7 @@ module Boxcars
|
|
10
10
|
# The default parameters to use when asking the engine.
|
11
11
|
DEFAULT_PARAMS = {
|
12
12
|
model: "gpt-3.5-turbo",
|
13
|
-
temperature: 0.
|
13
|
+
temperature: 0.2,
|
14
14
|
max_tokens: 512
|
15
15
|
}.freeze
|
16
16
|
|
@@ -38,17 +38,22 @@ module Boxcars
|
|
38
38
|
# @param openai_access_token [String] The access token to use when asking the engine.
|
39
39
|
# Defaults to Boxcars.configuration.openai_access_token.
|
40
40
|
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
41
|
-
def client(prompt:, openai_access_token: nil, **kwargs)
|
41
|
+
def client(prompt:, inputs: {}, openai_access_token: nil, **kwargs)
|
42
42
|
access_token = Boxcars.configuration.openai_access_token(openai_access_token: openai_access_token)
|
43
43
|
organization_id = Boxcars.configuration.organization_id
|
44
44
|
clnt = ::OpenAI::Client.new(access_token: access_token, organization_id: organization_id)
|
45
|
-
|
46
|
-
if
|
45
|
+
params = open_ai_params.merge(kwargs)
|
46
|
+
if params[:model] == "gpt-3.5-turbo"
|
47
47
|
prompt = prompt.first if prompt.is_a?(Array)
|
48
|
-
|
49
|
-
|
48
|
+
params = prompt.as_messages(inputs).merge(params)
|
49
|
+
if Boxcars.configuration.log_prompts
|
50
|
+
Boxcars.debug(params[:messages].map { |p| ">>>>>> Role: #{p[:role]} <<<<<<\n#{p[:content]}" }.join("\n"), :cyan)
|
51
|
+
end
|
52
|
+
clnt.chat(parameters: params)
|
50
53
|
else
|
51
|
-
|
54
|
+
params = prompt.as_prompt(inputs).merge(params)
|
55
|
+
Boxcars.debug("Prompt after formatting:\n#{params[:prompt]}", :cyan) if Boxcars.configuration.log_prompts
|
56
|
+
clnt.completions(parameters: params)
|
52
57
|
end
|
53
58
|
end
|
54
59
|
|
@@ -56,7 +61,8 @@ module Boxcars
|
|
56
61
|
# @param question [String] The question to ask the engine.
|
57
62
|
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
58
63
|
def run(question, **kwargs)
|
59
|
-
|
64
|
+
prompt = Prompt.new(template: question)
|
65
|
+
response = client(prompt: prompt, **kwargs)
|
60
66
|
answer = response["choices"].map { |c| c.dig("message", "content") || c["text"] }.join("\n").strip
|
61
67
|
puts answer
|
62
68
|
answer
|
@@ -110,6 +116,7 @@ module Boxcars
|
|
110
116
|
|
111
117
|
# Call out to OpenAI's endpoint with k unique prompts.
|
112
118
|
# @param prompts [Array<String>] The prompts to pass into the model.
|
119
|
+
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
113
120
|
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
114
121
|
# @return [EngineResult] The full engine output.
|
115
122
|
def generate(prompts:, stop: nil)
|
@@ -120,13 +127,14 @@ module Boxcars
|
|
120
127
|
# Get the token usage from the response.
|
121
128
|
# Includes prompt, completion, and total tokens used.
|
122
129
|
inkeys = %w[completion_tokens prompt_tokens total_tokens].freeze
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
+
prompts.each_slice(batch_size) do |sub_prompts|
|
131
|
+
sub_prompts.each do |sprompts, inputs|
|
132
|
+
response = client(prompt: sprompts, inputs: inputs, **params)
|
133
|
+
check_response(response)
|
134
|
+
choices.concat(response["choices"])
|
135
|
+
usage_keys = inkeys & response["usage"].keys
|
136
|
+
usage_keys.each { |key| token_usage[key] = token_usage[key].to_i + response["usage"][key] }
|
137
|
+
end
|
130
138
|
end
|
131
139
|
|
132
140
|
n = params.fetch(:n, 1)
|
data/lib/boxcars/prompt.rb
CHANGED
@@ -16,6 +16,31 @@ module Boxcars
|
|
16
16
|
@output_variables = output_variables || [:output]
|
17
17
|
end
|
18
18
|
|
19
|
+
# compute the prompt parameters with input substitutions (used for chatGPT)
|
20
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
21
|
+
# @return [Hash] The formatted prompt { messages: ...}
|
22
|
+
def as_prompt(inputs)
|
23
|
+
{ prompt: format(inputs) }
|
24
|
+
end
|
25
|
+
|
26
|
+
# compute the prompt parameters with input substitutions
|
27
|
+
# @param inputs [Hash] The inputs to use for the prompt.
|
28
|
+
# @return [Hash] The formatted prompt { prompt: "..."}
|
29
|
+
def as_messages(inputs)
|
30
|
+
{ messages: [{ role: :assistant, content: format(inputs) }] }
|
31
|
+
end
|
32
|
+
|
33
|
+
# tack on the ongoing conversation if present to the prompt
|
34
|
+
def with_conversation(conversation)
|
35
|
+
return self unless conversation
|
36
|
+
|
37
|
+
new_prompt = dup
|
38
|
+
new_prompt.template += "\n\n#{conversation.message_text}"
|
39
|
+
new_prompt
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
19
44
|
# format the prompt with the input variables
|
20
45
|
# @param inputs [Hash] The inputs to use for the prompt.
|
21
46
|
# @return [String] The formatted prompt.
|
@@ -27,43 +52,5 @@ module Boxcars
|
|
27
52
|
Boxcars.error "Missing prompt input key: #{first_line}"
|
28
53
|
raise KeyError, "Prompt format error: #{first_line}"
|
29
54
|
end
|
30
|
-
|
31
|
-
# check if the template is valid
|
32
|
-
def template_is_valid?
|
33
|
-
all_vars = (input_variables + other_inputs + output_variables).sort
|
34
|
-
template_vars = @template.scan(/%<(\w+)>s/).flatten.map(&:to_sym).sort
|
35
|
-
all_vars == template_vars
|
36
|
-
end
|
37
|
-
|
38
|
-
# missing variables in the template
|
39
|
-
def missing_variables?(inputs)
|
40
|
-
input_vars = [input_variables, other_inputs].flatten.sort
|
41
|
-
return if inputs.keys.sort == input_vars
|
42
|
-
|
43
|
-
raise ArgumentError, "Missing expected input keys, got: #{inputs.keys}. Expected: #{input_vars}"
|
44
|
-
end
|
45
|
-
|
46
|
-
# create a prompt template from examples
|
47
|
-
# @param examples [String] or [Array<String>] The example(s) to use for the template.
|
48
|
-
# @param input_variables [Array<Symbol>] The input variables to use for the prompt.
|
49
|
-
# @param example_separator [String] The separator to use between the examples. Defaults to "\n\n"
|
50
|
-
# @param prefix [String] The prefix to use for the template. Defaults to ""
|
51
|
-
def self.from_examples(examples:, suffix:, input_variables:, example_separator: "\n\n", prefix: "", **kwargs)
|
52
|
-
template = [prefix, examples, suffix].join(example_separator)
|
53
|
-
other_inputs = kwargs[:other_inputs] || []
|
54
|
-
output_variables = kwargs[:output_variables] || [:output]
|
55
|
-
Prompt.new(template: template, input_variables: input_variables, other_inputs: other_inputs,
|
56
|
-
output_variables: output_variables)
|
57
|
-
end
|
58
|
-
|
59
|
-
# create a prompt template from a file
|
60
|
-
# @param path [String] The path to the file to use for the template.
|
61
|
-
# @param input_variables [Array<Symbol>] The input variables to use for the prompt. Defaults to [:input]
|
62
|
-
# @param output_variables [Array<Symbol>] The output variables to use for the prompt. Defaults to [:output]
|
63
|
-
def self.from_file(path:, input_variables: nil, other_inputs: nil, output_variables: nil)
|
64
|
-
template = File.read(path)
|
65
|
-
Prompt.new(template: template, input_variables: input_variables, other_inputs: other_inputs,
|
66
|
-
output_variables: output_variables)
|
67
|
-
end
|
68
55
|
end
|
69
56
|
end
|
@@ -1,35 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Agent for the MRKL chain
|
2
4
|
module Boxcars
|
3
5
|
# A Train using the zero-shot react method.
|
4
6
|
class ZeroShot < Train
|
5
7
|
attr_reader :boxcars, :observation_prefix, :engine_prefix
|
6
8
|
|
7
|
-
# default prompt prefix
|
8
|
-
PREFIX = "Answer the following questions as best you can. You have access to the following actions:".freeze
|
9
|
-
|
10
|
-
# default prompt instructions
|
11
|
-
FORMAT_INSTRUCTIONS = <<~FINPUT.freeze
|
12
|
-
Use the following format:
|
13
|
-
|
14
|
-
Question: the input question you must answer
|
15
|
-
Thought: you should always think about what to do
|
16
|
-
Action: the action to take, should be one of [%<boxcar_names>s]
|
17
|
-
Action Input: the input to the action
|
18
|
-
Observation: the result of the action
|
19
|
-
... (this Thought/Action/Action Input/Observation sequence can repeat N times)
|
20
|
-
Thought: I now know the final answer
|
21
|
-
Final Answer: the final answer to the original input question
|
22
|
-
Next Actions: If you have them, up to three suggested actions for the user to take after getting this answer.
|
23
|
-
FINPUT
|
24
|
-
|
25
|
-
# default prompt suffix
|
26
|
-
SUFFIX = <<~SINPUT.freeze
|
27
|
-
Begin!
|
28
|
-
|
29
|
-
Question: %<input>s
|
30
|
-
Thought:%<agent_scratchpad>s
|
31
|
-
SINPUT
|
32
|
-
|
33
9
|
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
34
10
|
# @param engine [Boxcars::Engine] The engine to use for this train.
|
35
11
|
# @param name [String] The name of the train. Defaults to 'Zero Shot'.
|
@@ -38,26 +14,26 @@ module Boxcars
|
|
38
14
|
def initialize(boxcars:, engine: nil, name: 'Zero Shot', description: 'Zero Shot Train', prompt: nil)
|
39
15
|
@observation_prefix = 'Observation: '
|
40
16
|
@engine_prefix = 'Thought:'
|
41
|
-
prompt ||=
|
17
|
+
prompt ||= my_prompt
|
42
18
|
super(engine: engine, boxcars: boxcars, prompt: prompt, name: name, description: description)
|
43
19
|
end
|
44
20
|
|
45
|
-
#
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
#
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
template = [prefix, boxcar_strings, format_instructions, suffix].join("\n\n")
|
56
|
-
Prompt.new(template: template, input_variables: input_variables)
|
21
|
+
# @return Hash The additional variables for this boxcar.
|
22
|
+
def prediction_additional
|
23
|
+
{ boxcar_names: boxcar_names, boxcar_descriptions: boxcar_descriptions }.merge super
|
24
|
+
end
|
25
|
+
|
26
|
+
# Extract the boxcar and input from the engine output.
|
27
|
+
# @param text [String] The output from the engine.
|
28
|
+
# @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
|
29
|
+
def extract_boxcar_and_input(text)
|
30
|
+
get_action_and_input(engine_output: text)
|
57
31
|
end
|
58
32
|
|
33
|
+
private
|
34
|
+
|
59
35
|
# the final answer action string
|
60
|
-
FINAL_ANSWER_ACTION = "Final Answer:"
|
36
|
+
FINAL_ANSWER_ACTION = "Final Answer:"
|
61
37
|
|
62
38
|
# Parse out the action and input from the engine output.
|
63
39
|
# @param engine_output [String] The output from the engine.
|
@@ -70,14 +46,18 @@ module Boxcars
|
|
70
46
|
if engine_output.include?(FINAL_ANSWER_ACTION)
|
71
47
|
answer = engine_output.split(FINAL_ANSWER_ACTION).last.strip
|
72
48
|
Result.new(status: :ok, answer: answer, explanation: engine_output)
|
73
|
-
# ['Final Answer', answer]
|
74
49
|
else
|
75
50
|
# the thought should be the frist line here if it doesn't start with "Action:"
|
76
51
|
thought = engine_output.split(/\n+/).reject(&:empty?).first
|
77
|
-
Boxcars.debug("
|
78
|
-
regex = /Action: (?<action>.*)\
|
52
|
+
Boxcars.debug("Thought: #{thought}", :yellow)
|
53
|
+
regex = /Action: (?<action>.*)\n+Action Input: (?<action_input>.*)/
|
79
54
|
match = regex.match(engine_output)
|
80
|
-
|
55
|
+
|
56
|
+
# we have an unexpected output from the engine
|
57
|
+
unless match
|
58
|
+
return [:error, "You gave me an improperly fomatted answer - try again. For example, if you know the final anwwer, " \
|
59
|
+
"start with #{FINAL_ANSWER_ACTION.inspect}"]
|
60
|
+
end
|
81
61
|
|
82
62
|
action = match[:action].strip
|
83
63
|
action_input = match[:action_input].strip.delete_prefix('"').delete_suffix('"')
|
@@ -85,11 +65,41 @@ module Boxcars
|
|
85
65
|
end
|
86
66
|
end
|
87
67
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
68
|
+
CTEMPLATE = [
|
69
|
+
syst("Answer the following questions as best you can. You have access to the following actions:\n",
|
70
|
+
"%<boxcar_descriptions>s\n",
|
71
|
+
"Use the following format:\n",
|
72
|
+
"Question: the input question you must answer\n",
|
73
|
+
"Thought: you should always think about what to do\n",
|
74
|
+
"Action: the action to take, should be one from this list: %<boxcar_names>s\n",
|
75
|
+
"Action Input: the input to the action\n",
|
76
|
+
"Observation: the result of the action\n",
|
77
|
+
"... (this Thought/Action/Action Input/Observation sequence can repeat N times)\n",
|
78
|
+
"Thought: I know the final answer\n",
|
79
|
+
"Final Answer: the final answer to the original input question\n",
|
80
|
+
"Next Actions: Up to 3 logical suggested next questions for the user to ask after getting this answer.\n",
|
81
|
+
"Remember to start a line with \"Final Answer:\" to give me the final answer.\n",
|
82
|
+
"Begin!"),
|
83
|
+
user("Question: %<input>s"),
|
84
|
+
assi("Thought: %<agent_scratchpad>s")
|
85
|
+
].freeze
|
86
|
+
|
87
|
+
def boxcar_names
|
88
|
+
@boxcar_names ||= "[#{boxcars.map(&:name).join(', ')}]"
|
89
|
+
end
|
90
|
+
|
91
|
+
def boxcar_descriptions
|
92
|
+
@boxcar_descriptions ||= boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
|
93
|
+
end
|
94
|
+
|
95
|
+
# The prompt to use for the train.
|
96
|
+
def my_prompt
|
97
|
+
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
98
|
+
@my_prompt ||= ConversationPrompt.new(
|
99
|
+
conversation: @conversation,
|
100
|
+
input_variables: [:input],
|
101
|
+
other_inputs: [:boxcar_names, :boxcar_descriptions, :agent_scratchpad],
|
102
|
+
output_variables: [:answer])
|
93
103
|
end
|
94
104
|
end
|
95
105
|
end
|
data/lib/boxcars/train.rb
CHANGED
@@ -3,33 +3,31 @@
|
|
3
3
|
module Boxcars
|
4
4
|
# @abstract
|
5
5
|
class Train < EngineBoxcar
|
6
|
-
attr_reader :
|
7
|
-
:max_iterations, :early_stopping_method
|
6
|
+
attr_reader :boxcars, :return_values, :return_intermediate_steps,
|
7
|
+
:max_iterations, :early_stopping_method, :name_to_boxcar_map
|
8
8
|
|
9
9
|
# A Train will use a engine to run a series of boxcars.
|
10
|
-
# @param engine [Boxcars::Engine] The engine to use for this train.
|
11
10
|
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
12
|
-
# @param prompt [
|
11
|
+
# @param prompt [Boxcars::Prompt] The prompt to use.
|
12
|
+
# @param engine [Boxcars::Engine] The engine to use for this train.
|
13
|
+
# @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
|
13
14
|
# @abstract
|
14
15
|
def initialize(boxcars:, prompt:, engine: nil, **kwargs)
|
15
16
|
@boxcars = boxcars
|
16
|
-
@
|
17
|
+
@name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
|
17
18
|
@return_values = [:output]
|
18
|
-
@return_intermediate_steps = kwargs
|
19
|
-
@max_iterations = kwargs
|
20
|
-
@early_stopping_method = kwargs
|
19
|
+
@return_intermediate_steps = kwargs.delete(:return_intermediate_steps) || false
|
20
|
+
@max_iterations = kwargs.delete(:max_iterations) || 25
|
21
|
+
@early_stopping_method = kwargs.delete(:early_stopping_method) || "force"
|
22
|
+
kwargs[:stop] ||= ["\n#{observation_prefix}"]
|
21
23
|
|
22
|
-
super(prompt: prompt, engine: engine,
|
24
|
+
super(prompt: prompt, engine: engine, **kwargs)
|
23
25
|
end
|
24
26
|
|
25
27
|
# Extract the boxcar name and input from the text.
|
26
28
|
# @param text [String] The text to extract from.
|
27
29
|
def extract_boxcar_and_input(text)
|
28
|
-
|
29
|
-
|
30
|
-
# the stop strings list
|
31
|
-
def stop
|
32
|
-
["\n#{observation_prefix}"]
|
30
|
+
Result.new(status: :ok, answer: text, explanation: engine_output)
|
33
31
|
end
|
34
32
|
|
35
33
|
# build the scratchpad for the engine
|
@@ -48,16 +46,18 @@ module Boxcars
|
|
48
46
|
# @param full_inputs [Hash] The inputs to the engine.
|
49
47
|
# @return [Boxcars::Action] The next action.
|
50
48
|
def get_next_action(full_inputs)
|
51
|
-
full_output =
|
52
|
-
parsed_output =
|
53
|
-
|
49
|
+
full_output = ""
|
50
|
+
parsed_output = nil
|
51
|
+
loop do
|
54
52
|
full_inputs[:agent_scratchpad] += full_output
|
55
53
|
output = predict(**full_inputs)
|
56
54
|
full_output += output
|
57
55
|
parsed_output = extract_boxcar_and_input(full_output)
|
56
|
+
break unless parsed_output.nil?
|
58
57
|
end
|
59
58
|
if parsed_output.is_a?(Result)
|
60
59
|
TrainAction.from_result(boxcar: "Final Answer", result: parsed_output, log: full_output)
|
60
|
+
# elsif parsed_output[0] == "Error"
|
61
61
|
else
|
62
62
|
TrainAction.new(boxcar: parsed_output[0], boxcar_input: parsed_output[1], log: full_output)
|
63
63
|
end
|
@@ -69,8 +69,7 @@ module Boxcars
|
|
69
69
|
# @return [Boxcars::Action] Action specifying what boxcar to use.
|
70
70
|
def plan(intermediate_steps, **kwargs)
|
71
71
|
thoughts = construct_scratchpad(intermediate_steps)
|
72
|
-
|
73
|
-
full_inputs = kwargs.merge(new_inputs)
|
72
|
+
full_inputs = prediction_additional.merge(kwargs).merge(agent_scratchpad: thoughts)
|
74
73
|
action = get_next_action(full_inputs)
|
75
74
|
return TrainFinish.new({ output: action.boxcar_input }, log: action.log) if action.boxcar == finish_boxcar_name
|
76
75
|
|
@@ -187,7 +186,6 @@ module Boxcars
|
|
187
186
|
# @return [Hash] The output.
|
188
187
|
def call(inputs:)
|
189
188
|
prepare_for_new_call
|
190
|
-
name_to_boxcar_map = boxcars.to_h { |boxcar| [boxcar.name, boxcar] }
|
191
189
|
intermediate_steps = []
|
192
190
|
iterations = 0
|
193
191
|
while should_continue?(iterations)
|
@@ -202,6 +200,9 @@ module Boxcars
|
|
202
200
|
Boxcars.error "Error in #{boxcar.name} boxcar#call: #{e}", :red
|
203
201
|
observation = "Error - #{e}, correct and try again."
|
204
202
|
end
|
203
|
+
elsif output.boxcar == :error
|
204
|
+
observation = output.log
|
205
|
+
return_direct = false
|
205
206
|
else
|
206
207
|
observation = "#{output.boxcar} is not a valid boxcar, try another one."
|
207
208
|
return_direct = false
|
data/lib/boxcars/version.rb
CHANGED
data/lib/boxcars.rb
CHANGED
@@ -25,12 +25,13 @@ module Boxcars
|
|
25
25
|
# Configuration contains gem settings
|
26
26
|
class Configuration
|
27
27
|
attr_writer :openai_access_token, :serpapi_api_key
|
28
|
-
attr_accessor :organization_id, :logger, :log_prompts, :default_train, :default_engine
|
28
|
+
attr_accessor :organization_id, :logger, :log_prompts, :log_generated, :default_train, :default_engine
|
29
29
|
|
30
30
|
def initialize
|
31
31
|
@organization_id = nil
|
32
32
|
@logger = Rails.logger if defined?(Rails)
|
33
33
|
@log_prompts = ENV.fetch("LOG_PROMPTS", false)
|
34
|
+
@log_generated = ENV.fetch("LOG_GEN", false)
|
34
35
|
end
|
35
36
|
|
36
37
|
# @return [String] The OpenAI Access Token either from arg or env.
|
@@ -107,6 +108,7 @@ module Boxcars
|
|
107
108
|
end
|
108
109
|
|
109
110
|
# Logging system
|
111
|
+
# debug log
|
110
112
|
def self.debug(msg, color = nil, **options)
|
111
113
|
msg = colorize(msg.to_s, color, **options) if color
|
112
114
|
if logger
|
@@ -116,6 +118,7 @@ module Boxcars
|
|
116
118
|
end
|
117
119
|
end
|
118
120
|
|
121
|
+
# info log
|
119
122
|
def self.info(msg, color = nil, **options)
|
120
123
|
msg = colorize(msg.to_s, color, **options) if color
|
121
124
|
if logger
|
@@ -125,6 +128,7 @@ module Boxcars
|
|
125
128
|
end
|
126
129
|
end
|
127
130
|
|
131
|
+
# warn log
|
128
132
|
def self.warn(msg, color = nil, **options)
|
129
133
|
msg = colorize(msg.to_s, color, **options) if color
|
130
134
|
if logger
|
@@ -134,6 +138,7 @@ module Boxcars
|
|
134
138
|
end
|
135
139
|
end
|
136
140
|
|
141
|
+
# error log
|
137
142
|
def self.error(msg, color = nil, **options)
|
138
143
|
msg = colorize(msg.to_s, color, **options) if color
|
139
144
|
if logger
|
@@ -143,6 +148,7 @@ module Boxcars
|
|
143
148
|
end
|
144
149
|
end
|
145
150
|
|
151
|
+
# simple colorization
|
146
152
|
def self.colorize(str, color, **options)
|
147
153
|
background = options[:background] || options[:bg] || false
|
148
154
|
style = options[:style]
|
@@ -157,6 +163,8 @@ end
|
|
157
163
|
|
158
164
|
require "boxcars/version"
|
159
165
|
require "boxcars/prompt"
|
166
|
+
require "boxcars/conversation_prompt"
|
167
|
+
require "boxcars/conversation"
|
160
168
|
require "boxcars/generation"
|
161
169
|
require "boxcars/ruby_repl"
|
162
170
|
require "boxcars/engine"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: boxcars
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Francis Sullivan
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2023-03-
|
12
|
+
date: 2023-03-20 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: debug
|
@@ -109,6 +109,8 @@ files:
|
|
109
109
|
- lib/boxcars/boxcar/engine_boxcar.rb
|
110
110
|
- lib/boxcars/boxcar/google_search.rb
|
111
111
|
- lib/boxcars/boxcar/sql.rb
|
112
|
+
- lib/boxcars/conversation.rb
|
113
|
+
- lib/boxcars/conversation_prompt.rb
|
112
114
|
- lib/boxcars/engine.rb
|
113
115
|
- lib/boxcars/engine/engine_result.rb
|
114
116
|
- lib/boxcars/engine/openai.rb
|