boxcars 0.3.1 → 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -2
- data/Gemfile.lock +12 -1
- data/boxcars.gemspec +1 -0
- data/lib/boxcars/boxcar/active_record.rb +1 -1
- data/lib/boxcars/boxcar/url_text.rb +58 -0
- data/lib/boxcars/boxcar.rb +23 -5
- data/lib/boxcars/conversation.rb +5 -2
- data/lib/boxcars/train/xml_train.rb +107 -0
- data/lib/boxcars/train/xml_zero_shot.rb +60 -0
- data/lib/boxcars/train/zero_shot.rb +3 -19
- data/lib/boxcars/train.rb +54 -17
- data/lib/boxcars/version.rb +1 -1
- data/lib/boxcars/x_node.rb +75 -0
- data/lib/boxcars.rb +4 -0
- metadata +20 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 463f57f1436cfea29e0a60bbfce071afd355a5d0cadbe1d69f96adc7393eacd6
|
4
|
+
data.tar.gz: 282e528fb8cb8b532b621db5c2e7ce2b82c4607057c01f0feb1eaacfe98ba09b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c76d2772db0925f71779c0bac1a2e92af20bcf398cc6e7cf943edeadb2014426554fba607b034fd68ac88bd26f2d42eef114de57332e83a4fe76558a9b2f0cea
|
7
|
+
data.tar.gz: '0489fffef87c32fbf6a7a9af4fbca77ef991437353211174617b6ea8e229e49accc85c39af93414ce1628a4472cc208d8075570abba5e8269b30cfb66000a40e'
|
data/CHANGELOG.md
CHANGED
@@ -1,12 +1,17 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
## [
|
3
|
+
## [v0.3.1](https://github.com/BoxcarsAI/boxcars/tree/v0.3.1) (2023-07-01)
|
4
4
|
|
5
|
-
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.16...
|
5
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.2.16...v0.3.1)
|
6
6
|
|
7
7
|
**Closed issues:**
|
8
8
|
|
9
9
|
- Add a running log of prompts for debugging [\#99](https://github.com/BoxcarsAI/boxcars/issues/99)
|
10
|
+
- Anyway to create conversation? [\#73](https://github.com/BoxcarsAI/boxcars/issues/73)
|
11
|
+
|
12
|
+
**Merged pull requests:**
|
13
|
+
|
14
|
+
- now, when you call run on a train multiple times, it remembers the ru… [\#101](https://github.com/BoxcarsAI/boxcars/pull/101) ([francis](https://github.com/francis))
|
10
15
|
|
11
16
|
## [v0.2.16](https://github.com/BoxcarsAI/boxcars/tree/v0.2.16) (2023-06-26)
|
12
17
|
|
data/Gemfile.lock
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
boxcars (0.3.
|
4
|
+
boxcars (0.3.2)
|
5
5
|
google_search_results (~> 2.2)
|
6
6
|
gpt4all (~> 0.0.4)
|
7
7
|
hnswlib (~> 0.8)
|
8
|
+
nokogiri (~> 1.15)
|
8
9
|
pgvector (~> 0.2)
|
9
10
|
ruby-openai (~> 4.1)
|
10
11
|
|
@@ -103,6 +104,14 @@ GEM
|
|
103
104
|
netrc (0.11.0)
|
104
105
|
nio4r (2.5.9)
|
105
106
|
nio4r (2.5.9-java)
|
107
|
+
nokogiri (1.15.2-arm64-darwin)
|
108
|
+
racc (~> 1.4)
|
109
|
+
nokogiri (1.15.2-java)
|
110
|
+
racc (~> 1.4)
|
111
|
+
nokogiri (1.15.2-x86_64-darwin)
|
112
|
+
racc (~> 1.4)
|
113
|
+
nokogiri (1.15.2-x86_64-linux)
|
114
|
+
racc (~> 1.4)
|
106
115
|
octokit (4.25.1)
|
107
116
|
faraday (>= 1, < 3)
|
108
117
|
sawyer (~> 0.9)
|
@@ -120,6 +129,8 @@ GEM
|
|
120
129
|
protocol-hpack (~> 1.4)
|
121
130
|
protocol-http (~> 0.18)
|
122
131
|
public_suffix (5.0.1)
|
132
|
+
racc (1.7.1)
|
133
|
+
racc (1.7.1-java)
|
123
134
|
rainbow (3.1.1)
|
124
135
|
rake (13.0.6)
|
125
136
|
regexp_parser (2.8.0)
|
data/boxcars.gemspec
CHANGED
@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_dependency "google_search_results", "~> 2.2"
|
35
35
|
spec.add_dependency "gpt4all", "~> 0.0.4"
|
36
36
|
spec.add_dependency "hnswlib", "~> 0.8"
|
37
|
+
spec.add_dependency "nokogiri", "~> 1.15"
|
37
38
|
spec.add_dependency "pgvector", "~> 0.2"
|
38
39
|
spec.add_dependency "ruby-openai", "~> 4.1"
|
39
40
|
|
@@ -147,7 +147,7 @@ module Boxcars
|
|
147
147
|
end
|
148
148
|
|
149
149
|
def change_count(changes_code)
|
150
|
-
return 0
|
150
|
+
return 0 if changes_code.nil? || changes_code.empty? || changes_code =~ %r{^(None|N/A)$}i
|
151
151
|
|
152
152
|
rollback_after_running do
|
153
153
|
Boxcars.debug "computing change count with: #{changes_code}", :yellow
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Boxcars
|
4
|
+
# A Boxcar that reads text from a URL.
|
5
|
+
class URLText < Boxcar
|
6
|
+
# the description of this boxcar
|
7
|
+
DESC = "useful when you want to get text from a URL."
|
8
|
+
|
9
|
+
# implements a boxcar that uses the Google SerpAPI to get answers to questions.
|
10
|
+
# @param name [String] The name of the boxcar. Defaults to classname.
|
11
|
+
# @param description [String] A description of the boxcar. Defaults to SERPDESC.
|
12
|
+
def initialize(name: "FetchURL", description: DESC)
|
13
|
+
super(name: name, description: description)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Get text from a url.
|
17
|
+
# @param url [String] The url
|
18
|
+
# @return [String] The text for the url.
|
19
|
+
def run(url)
|
20
|
+
url = URI.parse(url)
|
21
|
+
get_answer(url)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def html_to_text(url, response)
|
27
|
+
Nokogiri::HTML(response.body).css(%w[h1 h2 h3 h4 h5 h6 p a].join(",")).map do |e|
|
28
|
+
itxt = e.inner_text.strip
|
29
|
+
itxt = itxt.gsub(/[[:space:]]+/, " ") # remove extra spaces
|
30
|
+
# next if itxt.nil? || itxt.empty?
|
31
|
+
if e.name == "a"
|
32
|
+
href = e.attributes["href"]&.value
|
33
|
+
href = URI.join(url, href).to_s if href =~ %r{^/}
|
34
|
+
"[#{itxt}](#{href})" # if e.attributes["href"]&.value =~ /^http/
|
35
|
+
else
|
36
|
+
itxt
|
37
|
+
end
|
38
|
+
end.compact.join("\n\n")
|
39
|
+
end
|
40
|
+
|
41
|
+
def get_answer(url)
|
42
|
+
response = Net::HTTP.get_response(url)
|
43
|
+
if response.is_a?(Net::HTTPSuccess)
|
44
|
+
return Result.from_text(response.body) if response.content_type == "text/plain"
|
45
|
+
|
46
|
+
if response.content_type == "text/html"
|
47
|
+
# return only the top level text
|
48
|
+
txt = html_to_text(url, response)
|
49
|
+
Result.from_text(txt)
|
50
|
+
else
|
51
|
+
Result.from_text(response.body)
|
52
|
+
end
|
53
|
+
else
|
54
|
+
Result.new(status: :error, explanation: "Error with url: #{response.code} #{response.message}")
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/boxcars/boxcar.rb
CHANGED
@@ -17,12 +17,12 @@ module Boxcars
|
|
17
17
|
|
18
18
|
# Input keys this chain expects.
|
19
19
|
def input_keys
|
20
|
-
|
20
|
+
[:question]
|
21
21
|
end
|
22
22
|
|
23
23
|
# Output keys this chain expects.
|
24
24
|
def output_keys
|
25
|
-
|
25
|
+
[:answer]
|
26
26
|
end
|
27
27
|
|
28
28
|
# Check that all inputs are present.
|
@@ -116,6 +116,20 @@ module Boxcars
|
|
116
116
|
end
|
117
117
|
# rubocop:enable Security/YAMLLoad
|
118
118
|
|
119
|
+
def schema
|
120
|
+
params = input_keys.map do |key|
|
121
|
+
"<param name=\"#{key}\" data-type=\"String\" required=\"true\" description=\"#{key}\" />"
|
122
|
+
end.join("\n")
|
123
|
+
<<~SCHEMA.freeze
|
124
|
+
<tool>
|
125
|
+
<tool name="#{name}" version="0.1" description="#{description}">
|
126
|
+
<params>
|
127
|
+
#{params}
|
128
|
+
</params>
|
129
|
+
</tool>
|
130
|
+
SCHEMA
|
131
|
+
end
|
132
|
+
|
119
133
|
private
|
120
134
|
|
121
135
|
# remember the history of this boxcar. Take the current intermediate steps and
|
@@ -126,18 +140,21 @@ module Boxcars
|
|
126
140
|
|
127
141
|
# insert conversation history into the prompt
|
128
142
|
history = []
|
129
|
-
history << Boxcar.user(
|
143
|
+
history << Boxcar.user(key_and_value_text(question_prefix, current_results[:input]))
|
130
144
|
current_results[:intermediate_steps].each do |action, obs|
|
131
145
|
if action.is_a?(TrainAction)
|
132
146
|
obs = Observation.new(status: :ok, note: obs) if obs.is_a?(String)
|
133
147
|
next if obs.status != :ok
|
134
148
|
|
135
|
-
history << Boxcar.assi("
|
149
|
+
history << Boxcar.assi("#{thought_prefix}#{action.log}", "\n",
|
150
|
+
key_and_value_text(observation_prefix, obs.note))
|
136
151
|
else
|
137
152
|
Boxcars.error "Unknown action: #{action}", :red
|
138
153
|
end
|
139
154
|
end
|
140
|
-
|
155
|
+
final_answer = key_and_value_text(final_answer_prefix, current_results[:output])
|
156
|
+
history << Boxcar.assi(
|
157
|
+
key_and_value_text(thought_prefix, "I know the final answer\n#{final_answer}\n"))
|
141
158
|
prompt.add_history(history)
|
142
159
|
end
|
143
160
|
|
@@ -196,6 +213,7 @@ require "boxcars/result"
|
|
196
213
|
require "boxcars/boxcar/engine_boxcar"
|
197
214
|
require "boxcars/boxcar/calculator"
|
198
215
|
require "boxcars/boxcar/google_search"
|
216
|
+
require "boxcars/boxcar/url_text"
|
199
217
|
require "boxcars/boxcar/wikipedia_search"
|
200
218
|
require "boxcars/boxcar/sql_base"
|
201
219
|
require "boxcars/boxcar/sql_active_record"
|
data/lib/boxcars/conversation.rb
CHANGED
@@ -64,12 +64,15 @@ module Boxcars
|
|
64
64
|
@lines += conversation.lines
|
65
65
|
end
|
66
66
|
|
67
|
-
# insert converation above history line
|
67
|
+
# insert converation above history line if it is present
|
68
68
|
# @param conversation [Conversation] The conversation to add
|
69
69
|
def add_history(conversation)
|
70
|
-
@lines = @lines.dup
|
71
70
|
# find the history line
|
72
71
|
hi = lines.rindex { |ln| ln[0] == :history }
|
72
|
+
return unless hi
|
73
|
+
|
74
|
+
@lines = @lines.dup
|
75
|
+
|
73
76
|
# insert the conversation above the history line
|
74
77
|
@lines.insert(hi, *conversation.lines)
|
75
78
|
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "nokogiri"
|
4
|
+
|
5
|
+
# base class for all XML trains
|
6
|
+
module Boxcars
|
7
|
+
# A Train using XML for prompting and execution.
|
8
|
+
class XMLTrain < Train
|
9
|
+
# A Train will use a engine to run a series of boxcars.
|
10
|
+
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
11
|
+
# @param prompt [Boxcars::Prompt] The prompt to use.
|
12
|
+
# @param engine [Boxcars::Engine] The engine to use for this train.
|
13
|
+
# @param kwargs [Hash] Additional arguments including: name, description, top_k, return_direct, and stop
|
14
|
+
# @abstract
|
15
|
+
def initialize(boxcars:, prompt:, engine: nil, **kwargs)
|
16
|
+
super
|
17
|
+
end
|
18
|
+
|
19
|
+
def init_prefixes
|
20
|
+
@thought_prefix ||= "<thought>"
|
21
|
+
@observation_prefix ||= "<observation>"
|
22
|
+
@final_answer_prefix ||= "<final_answer>"
|
23
|
+
@answer_prefix ||= "<answer>"
|
24
|
+
@question_prefix ||= "<question>"
|
25
|
+
@output_prefix ||= "<output>"
|
26
|
+
end
|
27
|
+
|
28
|
+
def close_tag(tag)
|
29
|
+
tag.to_s.sub("<", "</") if tag.to_s[0] == "<"
|
30
|
+
end
|
31
|
+
|
32
|
+
# the xml to describe the boxcars
|
33
|
+
def boxcars_xml
|
34
|
+
schema = boxcars.map(&:schema).join("\n")
|
35
|
+
"<boxcars>\n#{schema}</boxcars>"
|
36
|
+
end
|
37
|
+
|
38
|
+
# @return Hash The additional variables for this boxcar.
|
39
|
+
def prediction_additional(_inputs)
|
40
|
+
{ boxcars_xml: boxcars_xml, next_actions: next_actions }.merge super
|
41
|
+
end
|
42
|
+
|
43
|
+
def build_output(text)
|
44
|
+
if text =~ /#{close_tag(thought_prefix)}/
|
45
|
+
"<data>#{engine_prefix}#{text}</data>"
|
46
|
+
else
|
47
|
+
"<data>#{text}</data>"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Extract the boxcar and input from the engine output.
|
52
|
+
# @param text [String] The output from the engine.
|
53
|
+
# @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
|
54
|
+
def extract_boxcar_and_input(text)
|
55
|
+
get_action_and_input(engine_output: build_output(text))
|
56
|
+
rescue StandardError => e
|
57
|
+
Boxcars.debug("Error: #{e.message}", :red)
|
58
|
+
[:error, e.message]
|
59
|
+
end
|
60
|
+
|
61
|
+
private
|
62
|
+
|
63
|
+
def parse_output(engine_output)
|
64
|
+
doc = Nokogiri::XML("<data>#{engine_prefix}#{engine_output}\n</data>")
|
65
|
+
keys = doc.element_children.first.element_children.map(&:name).map(&:to_sym)
|
66
|
+
keys.to_h do |key|
|
67
|
+
[key, doc.at_xpath("//#{key}")&.text]
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
def child_keys(xnode)
|
72
|
+
xnode.children.map(&:name).map(&:to_sym)
|
73
|
+
end
|
74
|
+
|
75
|
+
# get next action and input using an XNode
|
76
|
+
# @param xnode [XNode] The XNode to use.
|
77
|
+
# @return [Array<String, String>] The action and input.
|
78
|
+
def xn_get_action_and_input(xnode)
|
79
|
+
action = xnode.xtext("//action")
|
80
|
+
action_input = xnode.xtext("//action_input")
|
81
|
+
thought = xnode.xtext("//thought")
|
82
|
+
final_answer = xnode.xtext("//final_answer")
|
83
|
+
|
84
|
+
# the thought should be the frist line here if it doesn't start with "Action:"
|
85
|
+
Boxcars.debug("Thought: #{thought}", :yellow)
|
86
|
+
|
87
|
+
if final_answer.present?
|
88
|
+
Result.new(status: :ok, answer: final_answer, explanation: final_answer)
|
89
|
+
else
|
90
|
+
# we have an unexpected output from the engine
|
91
|
+
unless action.present? && action_input.present?
|
92
|
+
return [:error, "You gave me an improperly formatted answer or didn't use tags."]
|
93
|
+
end
|
94
|
+
|
95
|
+
Boxcars.debug("Action: #{action}\nAction Input: #{action_input}", :yellow)
|
96
|
+
[action, action_input]
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
# Parse out the action and input from the engine output.
|
101
|
+
# @param engine_output [String] The output from the engine.
|
102
|
+
# @return [Array<String>] The action and input.
|
103
|
+
def get_action_and_input(engine_output:)
|
104
|
+
xn_get_action_and_input(XNode.from_xml(engine_output))
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "nokogiri"
|
4
|
+
|
5
|
+
# Agent for the MRKL chain
|
6
|
+
module Boxcars
|
7
|
+
# A Train using the zero-shot react method and only XML in the prompt.
|
8
|
+
class XMLZeroShot < XMLTrain
|
9
|
+
attr_reader :boxcars
|
10
|
+
attr_accessor :wants_next_actions
|
11
|
+
|
12
|
+
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
13
|
+
# @param engine [Boxcars::Engine] The engine to use for this train.
|
14
|
+
# @param name [String] The name of the train. Defaults to 'Zero Shot'.
|
15
|
+
# @param description [String] The description of the train. Defaults to 'Zero Shot Train'.
|
16
|
+
# @param prompt [Boxcars::Prompt] The prompt to use. Defaults to the built-in prompt.
|
17
|
+
# @param kwargs [Hash] Additional arguments to pass to the train. wants_next_actions: true
|
18
|
+
def initialize(boxcars:, engine: nil, name: 'Zero Shot XML', description: 'Zero Shot Train wiht XML', prompt: nil, **kwargs)
|
19
|
+
@engine_prefix = '<thought>'
|
20
|
+
@wants_next_actions = kwargs.fetch(:wants_next_actions, false)
|
21
|
+
prompt ||= my_prompt
|
22
|
+
super(engine: engine, boxcars: boxcars, prompt: prompt, name: name, description: description, **kwargs)
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
CTEMPLATE = [
|
28
|
+
syst("<training>Answer the following questions as best you can. You have access to the following tools for actions:\n",
|
29
|
+
"%<boxcars_xml>s",
|
30
|
+
"Use the following format making sure all open tags have closing tags:\n",
|
31
|
+
" <question>the input question you must answer</question>\n",
|
32
|
+
" <thought>you should always think about what to do</thought>\n",
|
33
|
+
" <action>the action to take, from this action list above</action>\n",
|
34
|
+
" <action_input>input to the action</action_input>\n",
|
35
|
+
" <observation>the result of the action</observation>\n",
|
36
|
+
" ... (this Thought/Action/Action Input/Observation sequence can repeat N times)\n",
|
37
|
+
" <thought>I know the final answer</thought>\n",
|
38
|
+
" <final_answer>the final answer to the original input question</final_answer>\n",
|
39
|
+
"-- FORMAT END -\n",
|
40
|
+
"Your answer should always have begin and end tags for each element.\n",
|
41
|
+
"Also make sure to specify a question for the action_input.\n",
|
42
|
+
"Finally, if you can deduct the answer from the question or observation, you can ",
|
43
|
+
"jump to final_answer and give me the answer.\n",
|
44
|
+
"</training>"),
|
45
|
+
hist, # insert thoughts here from previous runs
|
46
|
+
user("<question>%<input>s</question>"),
|
47
|
+
assi("<thought>%<agent_scratchpad>s")
|
48
|
+
].freeze
|
49
|
+
|
50
|
+
# The prompt to use for the train.
|
51
|
+
def my_prompt
|
52
|
+
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
53
|
+
@my_prompt ||= ConversationPrompt.new(
|
54
|
+
conversation: @conversation,
|
55
|
+
input_variables: [:input],
|
56
|
+
other_inputs: [:boxcars_xml, :next_actions, :agent_scratchpad],
|
57
|
+
output_variables: [:answer])
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -4,7 +4,7 @@
|
|
4
4
|
module Boxcars
|
5
5
|
# A Train using the zero-shot react method.
|
6
6
|
class ZeroShot < Train
|
7
|
-
attr_reader :boxcars, :observation_prefix
|
7
|
+
attr_reader :boxcars, :observation_prefix
|
8
8
|
attr_accessor :wants_next_actions
|
9
9
|
|
10
10
|
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
@@ -14,8 +14,6 @@ module Boxcars
|
|
14
14
|
# @param prompt [Boxcars::Prompt] The prompt to use. Defaults to the built-in prompt.
|
15
15
|
# @param kwargs [Hash] Additional arguments to pass to the train. wants_next_actions: true
|
16
16
|
def initialize(boxcars:, engine: nil, name: 'Zero Shot', description: 'Zero Shot Train', prompt: nil, **kwargs)
|
17
|
-
@observation_prefix = 'Observation: '
|
18
|
-
@engine_prefix = 'Thought:'
|
19
17
|
@wants_next_actions = kwargs.fetch(:wants_next_actions, false)
|
20
18
|
prompt ||= my_prompt
|
21
19
|
super(engine: engine, boxcars: boxcars, prompt: prompt, name: name, description: description, **kwargs)
|
@@ -31,6 +29,8 @@ module Boxcars
|
|
31
29
|
# @return [Array<Boxcars::Boxcar, String>] The boxcar and input.
|
32
30
|
def extract_boxcar_and_input(text)
|
33
31
|
get_action_and_input(engine_output: text)
|
32
|
+
rescue StandardError => e
|
33
|
+
[:error, e.message]
|
34
34
|
end
|
35
35
|
|
36
36
|
private
|
@@ -94,22 +94,6 @@ module Boxcars
|
|
94
94
|
assi("Thought: %<agent_scratchpad>s")
|
95
95
|
].freeze
|
96
96
|
|
97
|
-
def boxcar_names
|
98
|
-
@boxcar_names ||= "[#{boxcars.map(&:name).join(', ')}]"
|
99
|
-
end
|
100
|
-
|
101
|
-
def boxcar_descriptions
|
102
|
-
@boxcar_descriptions ||= boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
|
103
|
-
end
|
104
|
-
|
105
|
-
def next_actions
|
106
|
-
if wants_next_actions
|
107
|
-
"Next Actions: Up to 3 logical suggested next questions for the user to ask after getting this answer.\n"
|
108
|
-
else
|
109
|
-
""
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
97
|
# The prompt to use for the train.
|
114
98
|
def my_prompt
|
115
99
|
@conversation ||= Conversation.new(lines: CTEMPLATE)
|
data/lib/boxcars/train.rb
CHANGED
@@ -4,7 +4,8 @@ module Boxcars
|
|
4
4
|
# @abstract
|
5
5
|
class Train < EngineBoxcar
|
6
6
|
attr_reader :boxcars, :return_values, :return_intermediate_steps,
|
7
|
-
:max_iterations, :early_stopping_method, :name_to_boxcar_map
|
7
|
+
:max_iterations, :early_stopping_method, :name_to_boxcar_map,
|
8
|
+
:observation_prefix, :thought_prefix, :final_answer_prefix, :answer_prefix, :question_prefix, :engine_prefix
|
8
9
|
|
9
10
|
# A Train will use a engine to run a series of boxcars.
|
10
11
|
# @param boxcars [Array<Boxcars::Boxcar>] The boxcars to run.
|
@@ -20,12 +21,21 @@ module Boxcars
|
|
20
21
|
kwargs.delete(:return_intermediate_steps)
|
21
22
|
@max_iterations = kwargs.delete(:max_iterations) || 25
|
22
23
|
@early_stopping_method = kwargs.delete(:early_stopping_method) || "force"
|
23
|
-
|
24
|
+
init_prefixes
|
25
|
+
kwargs[:stop] = ["\n#{observation_prefix}"] unless kwargs.key?(:stop)
|
24
26
|
|
25
27
|
super(prompt: prompt, engine: engine, **kwargs)
|
26
28
|
end
|
27
29
|
|
28
|
-
|
30
|
+
def init_prefixes
|
31
|
+
@thought_prefix ||= "Thought: "
|
32
|
+
@observation_prefix ||= "Observation: "
|
33
|
+
@final_answer_prefix ||= "Final Answer: "
|
34
|
+
@answer_prefix ||= "Answer:"
|
35
|
+
@question_prefix ||= "Question: "
|
36
|
+
end
|
37
|
+
|
38
|
+
# Callback to process the action/action input of a train.
|
29
39
|
# @param text [String] The text to extract from.
|
30
40
|
def extract_boxcar_and_input(text)
|
31
41
|
Result.new(status: :ok, answer: text, explanation: engine_output)
|
@@ -34,16 +44,14 @@ module Boxcars
|
|
34
44
|
# build the scratchpad for the engine
|
35
45
|
# @param intermediate_steps [Array] The intermediate steps to build the scratchpad from.
|
36
46
|
# @return [String] The scratchpad.
|
37
|
-
# rubocop:disable Lint/RedundantStringCoercion
|
38
47
|
def construct_scratchpad(intermediate_steps)
|
39
48
|
thoughts = ""
|
40
49
|
intermediate_steps.each do |action, observation|
|
41
50
|
thoughts += action.is_a?(String) ? action : " #{action.log}"
|
42
|
-
thoughts += "\n#{
|
51
|
+
thoughts += "\n#{observation_text(observation)}\n#{engine_prefix}"
|
43
52
|
end
|
44
53
|
thoughts
|
45
54
|
end
|
46
|
-
# rubocop:enable Lint/RedundantStringCoercion
|
47
55
|
|
48
56
|
# determine the next action
|
49
57
|
# @param full_inputs [Hash] The inputs to the engine.
|
@@ -91,9 +99,7 @@ module Boxcars
|
|
91
99
|
# the input keys
|
92
100
|
# @return [Array<Symbol>] The input keys.
|
93
101
|
def input_keys
|
94
|
-
|
95
|
-
list.delete(:agent_scratchpad)
|
96
|
-
list
|
102
|
+
prompt.input_variables - [:agent_scratchpad]
|
97
103
|
end
|
98
104
|
|
99
105
|
# the output keys
|
@@ -123,13 +129,6 @@ module Boxcars
|
|
123
129
|
final_output
|
124
130
|
end
|
125
131
|
|
126
|
-
# the prefix for the engine
|
127
|
-
# @param return_direct [Boolean] Whether to return directly.
|
128
|
-
# @return [String] The prefix.
|
129
|
-
def engine_prefix(return_direct)
|
130
|
-
return_direct ? "" : engine_prefix
|
131
|
-
end
|
132
|
-
|
133
132
|
# validate the prompt
|
134
133
|
# @param values [Hash] The values to validate.
|
135
134
|
# @return [Hash] The validated values.
|
@@ -162,7 +161,7 @@ module Boxcars
|
|
162
161
|
thoughts = ""
|
163
162
|
intermediate_steps.each do |action, observation|
|
164
163
|
thoughts += action.log
|
165
|
-
thoughts += "\n#{
|
164
|
+
thoughts += "\n#{observation_text(observation)}\n#{engine_prefix}"
|
166
165
|
end
|
167
166
|
thoughts += "\n\nI now need to return a final answer based on the previous steps:"
|
168
167
|
new_inputs = { agent_scratchpad: thoughts, stop: _stop }
|
@@ -173,6 +172,7 @@ module Boxcars
|
|
173
172
|
TrainFinish.new({ output: full_output }, full_output)
|
174
173
|
else
|
175
174
|
boxcar, boxcar_input = parsed_output
|
175
|
+
Boxcars.debug "Got boxcar #{boxcar} and input #{boxcar_input}"
|
176
176
|
if boxcar == finish_boxcar_name
|
177
177
|
TrainFinish.new({ output: boxcar_input }, full_output)
|
178
178
|
else
|
@@ -225,9 +225,46 @@ module Boxcars
|
|
225
225
|
output = return_stopped_response(early_stopping_method, intermediate_steps, **inputs)
|
226
226
|
pre_return(output, intermediate_steps)
|
227
227
|
end
|
228
|
+
|
229
|
+
def key_and_value_text(key, value)
|
230
|
+
value = value.to_s
|
231
|
+
if key =~ /^<(?<tag_name>[[:word:]]+)>$/
|
232
|
+
# we need a close tag too
|
233
|
+
"#{key}#{value}</#{Regexp.last_match[:tag_name]}>"
|
234
|
+
else
|
235
|
+
"#{key}#{value}"
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
# this is for the scratchpad
|
240
|
+
def observation_text(observation)
|
241
|
+
key_and_value_text(observation_prefix, observation)
|
242
|
+
end
|
243
|
+
|
244
|
+
def question_text(question)
|
245
|
+
key_and_value_text(question_prefix, question)
|
246
|
+
end
|
247
|
+
|
248
|
+
def boxcar_names
|
249
|
+
@boxcar_names ||= boxcars.map(&:name).join(', ')
|
250
|
+
end
|
251
|
+
|
252
|
+
def boxcar_descriptions
|
253
|
+
@boxcar_descriptions ||= boxcars.map { |boxcar| "#{boxcar.name}: #{boxcar.description}" }.join("\n")
|
254
|
+
end
|
255
|
+
|
256
|
+
def next_actions
|
257
|
+
if wants_next_actions
|
258
|
+
"Next Actions: Up to 3 logical suggested next questions for the user to ask after getting this answer.\n"
|
259
|
+
else
|
260
|
+
""
|
261
|
+
end
|
262
|
+
end
|
228
263
|
end
|
229
264
|
end
|
230
265
|
|
231
266
|
require "boxcars/train/train_action"
|
232
267
|
require "boxcars/train/train_finish"
|
233
268
|
require "boxcars/train/zero_shot"
|
269
|
+
require "boxcars/train/xml_train"
|
270
|
+
require "boxcars/train/xml_zero_shot"
|
data/lib/boxcars/version.rb
CHANGED
@@ -0,0 +1,75 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'nokogiri'
|
4
|
+
|
5
|
+
module Boxcars
|
6
|
+
class XNode
|
7
|
+
attr_accessor :node, :children, :attributes
|
8
|
+
|
9
|
+
def initialize(node)
|
10
|
+
@node = node
|
11
|
+
@valid_names = []
|
12
|
+
@children = {}
|
13
|
+
# @attributes = node.attributes.transform_values(&:value)
|
14
|
+
@attributes = node.attributes.values.to_h { |a| [a.name.to_sym, a.value] }
|
15
|
+
|
16
|
+
node.children.each do |child|
|
17
|
+
next if child.text?
|
18
|
+
|
19
|
+
child_node = XNode.new(child)
|
20
|
+
if @children[child.name].nil?
|
21
|
+
@valid_names << child.name.to_sym
|
22
|
+
@children[child.name] = child_node
|
23
|
+
elsif @children[child.name].is_a?(Array)
|
24
|
+
@children[child.name] << child_node
|
25
|
+
else
|
26
|
+
@children[child.name] = [@children[child.name], child_node]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.from_xml(xml)
|
32
|
+
doc = Nokogiri::XML.parse(xml)
|
33
|
+
raise XmlError, "XML is not valid: #{doc.errors.map { |e| "#{e.line}:#{e.column} #{e.message}" }}" if doc.errors.any?
|
34
|
+
|
35
|
+
XNode.new(doc.root)
|
36
|
+
end
|
37
|
+
|
38
|
+
def xml
|
39
|
+
@node.to_xml
|
40
|
+
end
|
41
|
+
|
42
|
+
def text
|
43
|
+
@node.text
|
44
|
+
end
|
45
|
+
|
46
|
+
def xpath(path)
|
47
|
+
@node.xpath(path)
|
48
|
+
end
|
49
|
+
|
50
|
+
def xtext(path)
|
51
|
+
rv = xpath(path)&.text&.gsub(/[[:space:]]+/, " ")&.strip
|
52
|
+
return nil if rv.empty?
|
53
|
+
|
54
|
+
rv
|
55
|
+
end
|
56
|
+
|
57
|
+
def stext
|
58
|
+
@stext ||= text.gsub(/[[:space:]]+/, " ").strip # remove extra spaces
|
59
|
+
end
|
60
|
+
|
61
|
+
def [](key)
|
62
|
+
@children[key.to_s]
|
63
|
+
end
|
64
|
+
|
65
|
+
def method_missing(name, *args)
|
66
|
+
return @children[name.to_s] if @children.key?(name.to_s)
|
67
|
+
|
68
|
+
super
|
69
|
+
end
|
70
|
+
|
71
|
+
def respond_to_missing?(method_name, include_private = false)
|
72
|
+
@valid_names.include?(method) || super
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
data/lib/boxcars.rb
CHANGED
@@ -22,6 +22,9 @@ module Boxcars
|
|
22
22
|
# Error class for all Boxcars key errors.
|
23
23
|
class KeyError < Error; end
|
24
24
|
|
25
|
+
# Error class for all Boxcars XML errors.
|
26
|
+
class XmlError < Error; end
|
27
|
+
|
25
28
|
# Configuration contains gem settings
|
26
29
|
class Configuration
|
27
30
|
attr_writer :openai_access_token, :serpapi_api_key
|
@@ -179,6 +182,7 @@ module Boxcars
|
|
179
182
|
end
|
180
183
|
|
181
184
|
require "boxcars/version"
|
185
|
+
require "boxcars/x_node"
|
182
186
|
require "boxcars/prompt"
|
183
187
|
require "boxcars/conversation_prompt"
|
184
188
|
require "boxcars/conversation"
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: boxcars
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Francis Sullivan
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2023-07-
|
12
|
+
date: 2023-07-10 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: google_search_results
|
@@ -53,6 +53,20 @@ dependencies:
|
|
53
53
|
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
55
|
version: '0.8'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: nokogiri
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '1.15'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '1.15'
|
56
70
|
- !ruby/object:Gem::Dependency
|
57
71
|
name: pgvector
|
58
72
|
requirement: !ruby/object:Gem::Requirement
|
@@ -113,6 +127,7 @@ files:
|
|
113
127
|
- lib/boxcars/boxcar/sql_base.rb
|
114
128
|
- lib/boxcars/boxcar/sql_sequel.rb
|
115
129
|
- lib/boxcars/boxcar/swagger.rb
|
130
|
+
- lib/boxcars/boxcar/url_text.rb
|
116
131
|
- lib/boxcars/boxcar/vector_answer.rb
|
117
132
|
- lib/boxcars/boxcar/wikipedia_search.rb
|
118
133
|
- lib/boxcars/conversation.rb
|
@@ -129,6 +144,8 @@ files:
|
|
129
144
|
- lib/boxcars/train.rb
|
130
145
|
- lib/boxcars/train/train_action.rb
|
131
146
|
- lib/boxcars/train/train_finish.rb
|
147
|
+
- lib/boxcars/train/xml_train.rb
|
148
|
+
- lib/boxcars/train/xml_zero_shot.rb
|
132
149
|
- lib/boxcars/train/zero_shot.rb
|
133
150
|
- lib/boxcars/vector_search.rb
|
134
151
|
- lib/boxcars/vector_store.rb
|
@@ -148,6 +165,7 @@ files:
|
|
148
165
|
- lib/boxcars/vector_store/pgvector/search.rb
|
149
166
|
- lib/boxcars/vector_store/split_text.rb
|
150
167
|
- lib/boxcars/version.rb
|
168
|
+
- lib/boxcars/x_node.rb
|
151
169
|
homepage: https://github.com/BoxcarsAI/boxcars
|
152
170
|
licenses:
|
153
171
|
- MIT
|