boxcars 0.4.9 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/Gemfile +7 -7
- data/Gemfile.lock +70 -62
- data/README.md +1 -1
- data/boxcars.gemspec +1 -1
- data/lib/boxcars/boxcar/active_record.rb +4 -4
- data/lib/boxcars/boxcar/google_search.rb +2 -2
- data/lib/boxcars/boxcar/ruby_calculator.rb +1 -1
- data/lib/boxcars/boxcar/sql_active_record.rb +1 -1
- data/lib/boxcars/boxcar/sql_sequel.rb +1 -1
- data/lib/boxcars/boxcar/url_text.rb +1 -1
- data/lib/boxcars/boxcar/wikipedia_search.rb +1 -1
- data/lib/boxcars/engine/cohere.rb +187 -0
- data/lib/boxcars/engine.rb +1 -0
- data/lib/boxcars/vector_store.rb +2 -3
- data/lib/boxcars/version.rb +1 -1
- data/lib/boxcars.rb +5 -0
- metadata +5 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 03363b9d82506ac0c784569089c21c605fd76cc0f11fbfdbb05744e1bc330ddd
|
|
4
|
+
data.tar.gz: 6fb219328727d9603bd86358a28e3f715e861db57ff74e49c98d94801b960651
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 56c9344b6e8c9fbf07323add0e4b3ee245ff6663331a5fd6db917152f7c2aadb428565988c6d63fdb25f5bf2fac2870f47bcf2e8d8fcd9f22239571ea7ee635e
|
|
7
|
+
data.tar.gz: 99f92d94254d6df4260b975642e3d58af6093f024917e3d0e082c8cefd15f94aed879bdc713536f492cb1942d6ffbf52c97b598bb0afa5b26d643c93f3fea132
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [v0.4.10](https://github.com/BoxcarsAI/boxcars/tree/v0.4.10) (2024-04-19)
|
|
4
|
+
|
|
5
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.4.9...v0.4.10)
|
|
6
|
+
|
|
7
|
+
**Merged pull requests:**
|
|
8
|
+
|
|
9
|
+
- Add llms [\#169](https://github.com/BoxcarsAI/boxcars/pull/169) ([francis](https://github.com/francis))
|
|
10
|
+
- \[infra\] Bump pg from 1.5.5 to 1.5.6 [\#163](https://github.com/BoxcarsAI/boxcars/pull/163) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
11
|
+
- \[infra\] Bump rubocop-rspec from 2.26.1 to 2.29.1 [\#160](https://github.com/BoxcarsAI/boxcars/pull/160) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
12
|
+
- \[infra\] Bump rdoc from 6.6.2 to 6.6.3.1 [\#158](https://github.com/BoxcarsAI/boxcars/pull/158) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
13
|
+
- \[infra\] Bump dotenv from 3.0.2 to 3.1.0 [\#152](https://github.com/BoxcarsAI/boxcars/pull/152) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
14
|
+
- \[infra\] Bump webmock from 3.22.0 to 3.23.0 [\#150](https://github.com/BoxcarsAI/boxcars/pull/150) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
15
|
+
- \[infra\] Bump webmock from 3.20.0 to 3.22.0 [\#148](https://github.com/BoxcarsAI/boxcars/pull/148) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
16
|
+
- \[infra\] Bump dotenv from 2.8.1 to 3.0.2 [\#146](https://github.com/BoxcarsAI/boxcars/pull/146) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
17
|
+
- \[infra\] Bump pg from 1.5.4 to 1.5.5 [\#144](https://github.com/BoxcarsAI/boxcars/pull/144) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
18
|
+
- \[infra\] Bump webmock from 3.19.1 to 3.20.0 [\#142](https://github.com/BoxcarsAI/boxcars/pull/142) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
19
|
+
- \[infra\] Bump nokogiri from 1.16.0 to 1.16.2 [\#141](https://github.com/BoxcarsAI/boxcars/pull/141) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
20
|
+
- \[infra\] Bump rspec from 3.12.0 to 3.13.0 [\#140](https://github.com/BoxcarsAI/boxcars/pull/140) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
21
|
+
- \[infra\] Bump sqlite3 from 1.7.1 to 1.7.2 [\#139](https://github.com/BoxcarsAI/boxcars/pull/139) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
22
|
+
|
|
23
|
+
## [v0.4.9](https://github.com/BoxcarsAI/boxcars/tree/v0.4.9) (2024-01-25)
|
|
24
|
+
|
|
25
|
+
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.4.8...v0.4.9)
|
|
26
|
+
|
|
27
|
+
**Merged pull requests:**
|
|
28
|
+
|
|
29
|
+
- use newest models by default from Open AI [\#138](https://github.com/BoxcarsAI/boxcars/pull/138) ([francis](https://github.com/francis))
|
|
30
|
+
- \[infra\] Bump sqlite3 from 1.7.0 to 1.7.1 [\#137](https://github.com/BoxcarsAI/boxcars/pull/137) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
31
|
+
- \[infra\] Bump rubocop from 1.60.0 to 1.60.2 [\#136](https://github.com/BoxcarsAI/boxcars/pull/136) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
32
|
+
- \[infra\] Bump rubocop from 1.59.0 to 1.60.0 [\#133](https://github.com/BoxcarsAI/boxcars/pull/133) ([dependabot[bot]](https://github.com/apps/dependabot))
|
|
33
|
+
|
|
3
34
|
## [v0.4.8](https://github.com/BoxcarsAI/boxcars/tree/v0.4.8) (2024-01-08)
|
|
4
35
|
|
|
5
36
|
[Full Changelog](https://github.com/BoxcarsAI/boxcars/compare/v0.4.7...v0.4.8)
|
data/Gemfile
CHANGED
|
@@ -7,13 +7,13 @@ gemspec
|
|
|
7
7
|
|
|
8
8
|
gem "debug", "~> 1.9"
|
|
9
9
|
|
|
10
|
-
gem "dotenv", "~>
|
|
10
|
+
gem "dotenv", "~> 3.1"
|
|
11
11
|
|
|
12
|
-
gem "rake", "~> 13.
|
|
12
|
+
gem "rake", "~> 13.2"
|
|
13
13
|
|
|
14
14
|
gem "sqlite3", "~> 1.7"
|
|
15
15
|
|
|
16
|
-
gem "async", "~>1.
|
|
16
|
+
gem "async", "~>1.32.1"
|
|
17
17
|
|
|
18
18
|
gem "activerecord", "~> 7.0"
|
|
19
19
|
|
|
@@ -31,10 +31,10 @@ gem "pg", "~> 1.5"
|
|
|
31
31
|
gem "pgvector", "~> 0.2.2"
|
|
32
32
|
|
|
33
33
|
group :development, :test do
|
|
34
|
-
gem "rspec", "~> 3.
|
|
35
|
-
gem "rubocop", "~> 1.
|
|
34
|
+
gem "rspec", "~> 3.13"
|
|
35
|
+
gem "rubocop", "~> 1.64"
|
|
36
36
|
gem "vcr", "~> 6.2.0"
|
|
37
|
-
gem "webmock", "~> 3.
|
|
37
|
+
gem "webmock", "~> 3.23.1"
|
|
38
38
|
gem "rubocop-rake", "~> 0.6.0"
|
|
39
|
-
gem "rubocop-rspec", "~> 2.
|
|
39
|
+
gem "rubocop-rspec", "~> 2.30"
|
|
40
40
|
end
|
data/Gemfile.lock
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
boxcars (0.
|
|
4
|
+
boxcars (0.5.1)
|
|
5
5
|
anthropic (~> 0.1)
|
|
6
6
|
google_search_results (~> 2.2)
|
|
7
7
|
gpt4all (~> 0.0.4)
|
|
8
8
|
hnswlib (~> 0.8)
|
|
9
9
|
nokogiri (~> 1.16)
|
|
10
10
|
pgvector (~> 0.2)
|
|
11
|
-
ruby-openai (>= 4.2, <
|
|
11
|
+
ruby-openai (>= 4.2, < 8.0)
|
|
12
12
|
|
|
13
13
|
GEM
|
|
14
14
|
remote: https://rubygems.org/
|
|
@@ -25,11 +25,11 @@ GEM
|
|
|
25
25
|
tzinfo (~> 2.0)
|
|
26
26
|
addressable (2.8.6)
|
|
27
27
|
public_suffix (>= 2.0.2, < 6.0)
|
|
28
|
-
anthropic (0.
|
|
28
|
+
anthropic (0.2.0)
|
|
29
29
|
faraday (>= 1)
|
|
30
30
|
faraday-multipart (>= 1)
|
|
31
31
|
ast (2.4.2)
|
|
32
|
-
async (1.
|
|
32
|
+
async (1.32.1)
|
|
33
33
|
console (~> 1.10)
|
|
34
34
|
nio4r (~> 2.3)
|
|
35
35
|
timers (~> 4.1)
|
|
@@ -48,31 +48,32 @@ GEM
|
|
|
48
48
|
async
|
|
49
49
|
async-pool (0.4.0)
|
|
50
50
|
async (>= 1.25)
|
|
51
|
-
|
|
51
|
+
bigdecimal (3.1.8)
|
|
52
52
|
concurrent-ruby (1.2.2)
|
|
53
|
-
console (1.
|
|
53
|
+
console (1.24.0)
|
|
54
54
|
fiber-annotation
|
|
55
55
|
fiber-local
|
|
56
|
-
|
|
56
|
+
json
|
|
57
|
+
crack (1.0.0)
|
|
58
|
+
bigdecimal
|
|
57
59
|
rexml
|
|
58
|
-
debug (1.9.
|
|
60
|
+
debug (1.9.2)
|
|
59
61
|
irb (~> 1.10)
|
|
60
62
|
reline (>= 0.3.8)
|
|
61
|
-
diff-lcs (1.5.
|
|
63
|
+
diff-lcs (1.5.1)
|
|
62
64
|
domain_name (0.5.20190701)
|
|
63
65
|
unf (>= 0.0.5, < 1.0.0)
|
|
64
|
-
dotenv (
|
|
66
|
+
dotenv (3.1.2)
|
|
65
67
|
event_stream_parser (1.0.0)
|
|
66
|
-
faraday (2.
|
|
67
|
-
|
|
68
|
-
faraday-net_http (>= 2.0, < 3.1)
|
|
69
|
-
ruby2_keywords (>= 0.0.4)
|
|
68
|
+
faraday (2.9.0)
|
|
69
|
+
faraday-net_http (>= 2.0, < 3.2)
|
|
70
70
|
faraday-http-cache (2.5.0)
|
|
71
71
|
faraday (>= 0.8)
|
|
72
72
|
faraday-multipart (1.0.4)
|
|
73
73
|
multipart-post (~> 2)
|
|
74
|
-
faraday-net_http (3.0
|
|
75
|
-
|
|
74
|
+
faraday-net_http (3.1.0)
|
|
75
|
+
net-http
|
|
76
|
+
faraday-retry (2.2.1)
|
|
76
77
|
faraday (~> 2.0)
|
|
77
78
|
fiber-annotation (0.2.0)
|
|
78
79
|
fiber-local (1.0.0)
|
|
@@ -97,33 +98,35 @@ GEM
|
|
|
97
98
|
domain_name (~> 0.5)
|
|
98
99
|
i18n (1.14.1)
|
|
99
100
|
concurrent-ruby (~> 1.0)
|
|
100
|
-
io-console (0.7.
|
|
101
|
-
irb (1.
|
|
102
|
-
rdoc
|
|
103
|
-
reline (>= 0.
|
|
104
|
-
json (2.7.
|
|
101
|
+
io-console (0.7.2)
|
|
102
|
+
irb (1.13.1)
|
|
103
|
+
rdoc (>= 4.0.0)
|
|
104
|
+
reline (>= 0.4.2)
|
|
105
|
+
json (2.7.2)
|
|
105
106
|
language_server-protocol (3.17.0.3)
|
|
106
107
|
mime-types (3.4.1)
|
|
107
108
|
mime-types-data (~> 3.2015)
|
|
108
109
|
mime-types-data (3.2023.0218.1)
|
|
109
110
|
minitest (5.20.0)
|
|
110
111
|
multi_json (1.15.0)
|
|
111
|
-
multipart-post (2.
|
|
112
|
+
multipart-post (2.4.1)
|
|
113
|
+
net-http (0.4.1)
|
|
114
|
+
uri
|
|
112
115
|
netrc (0.11.0)
|
|
113
|
-
nio4r (2.
|
|
114
|
-
nokogiri (1.16.
|
|
116
|
+
nio4r (2.7.1)
|
|
117
|
+
nokogiri (1.16.5-arm64-darwin)
|
|
115
118
|
racc (~> 1.4)
|
|
116
|
-
nokogiri (1.16.
|
|
119
|
+
nokogiri (1.16.5-x86_64-linux)
|
|
117
120
|
racc (~> 1.4)
|
|
118
121
|
octokit (4.25.1)
|
|
119
122
|
faraday (>= 1, < 3)
|
|
120
123
|
sawyer (~> 0.9)
|
|
121
124
|
os (1.1.4)
|
|
122
125
|
parallel (1.24.0)
|
|
123
|
-
parser (3.3.0
|
|
126
|
+
parser (3.3.2.0)
|
|
124
127
|
ast (~> 2.4.1)
|
|
125
128
|
racc
|
|
126
|
-
pg (1.5.
|
|
129
|
+
pg (1.5.6)
|
|
127
130
|
pgvector (0.2.2)
|
|
128
131
|
protocol-hpack (1.4.2)
|
|
129
132
|
protocol-http (0.25.0)
|
|
@@ -134,35 +137,36 @@ GEM
|
|
|
134
137
|
protocol-http (~> 0.18)
|
|
135
138
|
psych (5.1.2)
|
|
136
139
|
stringio
|
|
137
|
-
public_suffix (5.0.
|
|
138
|
-
racc (1.
|
|
140
|
+
public_suffix (5.0.5)
|
|
141
|
+
racc (1.8.0)
|
|
139
142
|
rainbow (3.1.1)
|
|
140
|
-
rake (13.1
|
|
141
|
-
rdoc (6.6.
|
|
143
|
+
rake (13.2.1)
|
|
144
|
+
rdoc (6.6.3.1)
|
|
142
145
|
psych (>= 4.0.0)
|
|
143
|
-
regexp_parser (2.9.
|
|
144
|
-
reline (0.
|
|
146
|
+
regexp_parser (2.9.2)
|
|
147
|
+
reline (0.5.6)
|
|
145
148
|
io-console (~> 0.5)
|
|
146
149
|
rest-client (2.1.0)
|
|
147
150
|
http-accept (>= 1.7.0, < 2.0)
|
|
148
151
|
http-cookie (>= 1.0.2, < 2.0)
|
|
149
152
|
mime-types (>= 1.16, < 4.0)
|
|
150
153
|
netrc (~> 0.8)
|
|
151
|
-
rexml (3.2.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
rspec-
|
|
155
|
-
rspec-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
154
|
+
rexml (3.2.8)
|
|
155
|
+
strscan (>= 3.0.9)
|
|
156
|
+
rspec (3.13.0)
|
|
157
|
+
rspec-core (~> 3.13.0)
|
|
158
|
+
rspec-expectations (~> 3.13.0)
|
|
159
|
+
rspec-mocks (~> 3.13.0)
|
|
160
|
+
rspec-core (3.13.0)
|
|
161
|
+
rspec-support (~> 3.13.0)
|
|
162
|
+
rspec-expectations (3.13.0)
|
|
159
163
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
160
|
-
rspec-support (~> 3.
|
|
161
|
-
rspec-mocks (3.
|
|
164
|
+
rspec-support (~> 3.13.0)
|
|
165
|
+
rspec-mocks (3.13.0)
|
|
162
166
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
163
|
-
rspec-support (~> 3.
|
|
164
|
-
rspec-support (3.
|
|
165
|
-
rubocop (1.
|
|
167
|
+
rspec-support (~> 3.13.0)
|
|
168
|
+
rspec-support (3.13.1)
|
|
169
|
+
rubocop (1.64.1)
|
|
166
170
|
json (~> 2.3)
|
|
167
171
|
language_server-protocol (>= 3.17.0)
|
|
168
172
|
parallel (~> 1.10)
|
|
@@ -170,34 +174,37 @@ GEM
|
|
|
170
174
|
rainbow (>= 2.2.2, < 4.0)
|
|
171
175
|
regexp_parser (>= 1.8, < 3.0)
|
|
172
176
|
rexml (>= 3.2.5, < 4.0)
|
|
173
|
-
rubocop-ast (>= 1.
|
|
177
|
+
rubocop-ast (>= 1.31.1, < 2.0)
|
|
174
178
|
ruby-progressbar (~> 1.7)
|
|
175
179
|
unicode-display_width (>= 2.4.0, < 3.0)
|
|
176
|
-
rubocop-ast (1.
|
|
177
|
-
parser (>= 3.
|
|
180
|
+
rubocop-ast (1.31.3)
|
|
181
|
+
parser (>= 3.3.1.0)
|
|
178
182
|
rubocop-capybara (2.20.0)
|
|
179
183
|
rubocop (~> 1.41)
|
|
180
184
|
rubocop-factory_bot (2.25.1)
|
|
181
185
|
rubocop (~> 1.41)
|
|
182
186
|
rubocop-rake (0.6.0)
|
|
183
187
|
rubocop (~> 1.0)
|
|
184
|
-
rubocop-rspec (2.
|
|
188
|
+
rubocop-rspec (2.30.0)
|
|
185
189
|
rubocop (~> 1.40)
|
|
186
190
|
rubocop-capybara (~> 2.17)
|
|
187
191
|
rubocop-factory_bot (~> 2.22)
|
|
188
|
-
|
|
192
|
+
rubocop-rspec_rails (~> 2.28)
|
|
193
|
+
rubocop-rspec_rails (2.28.3)
|
|
194
|
+
rubocop (~> 1.40)
|
|
195
|
+
ruby-openai (7.0.1)
|
|
189
196
|
event_stream_parser (>= 0.3.0, < 2.0.0)
|
|
190
197
|
faraday (>= 1)
|
|
191
198
|
faraday-multipart (>= 1)
|
|
192
199
|
ruby-progressbar (1.13.0)
|
|
193
|
-
ruby2_keywords (0.0.5)
|
|
194
200
|
sawyer (0.9.2)
|
|
195
201
|
addressable (>= 2.3.5)
|
|
196
202
|
faraday (>= 0.17.3, < 3)
|
|
197
|
-
sqlite3 (1.7.
|
|
198
|
-
sqlite3 (1.7.
|
|
203
|
+
sqlite3 (1.7.2-arm64-darwin)
|
|
204
|
+
sqlite3 (1.7.2-x86_64-linux)
|
|
199
205
|
stringio (3.1.0)
|
|
200
206
|
strings-ansi (0.2.0)
|
|
207
|
+
strscan (3.1.0)
|
|
201
208
|
timers (4.3.5)
|
|
202
209
|
traces (0.11.1)
|
|
203
210
|
tty-cursor (0.7.1)
|
|
@@ -213,8 +220,9 @@ GEM
|
|
|
213
220
|
unf_ext
|
|
214
221
|
unf_ext (0.0.8.2)
|
|
215
222
|
unicode-display_width (2.5.0)
|
|
223
|
+
uri (0.13.0)
|
|
216
224
|
vcr (6.2.0)
|
|
217
|
-
webmock (3.
|
|
225
|
+
webmock (3.23.1)
|
|
218
226
|
addressable (>= 2.8.0)
|
|
219
227
|
crack (>= 0.3.2)
|
|
220
228
|
hashdiff (>= 0.4.0, < 2.0.0)
|
|
@@ -227,24 +235,24 @@ PLATFORMS
|
|
|
227
235
|
DEPENDENCIES
|
|
228
236
|
activerecord (~> 7.0)
|
|
229
237
|
activesupport (~> 7.0)
|
|
230
|
-
async (~> 1.
|
|
238
|
+
async (~> 1.32.1)
|
|
231
239
|
boxcars!
|
|
232
240
|
debug (~> 1.9)
|
|
233
|
-
dotenv (~>
|
|
241
|
+
dotenv (~> 3.1)
|
|
234
242
|
faraday-retry (~> 2.0)
|
|
235
243
|
github_changelog_generator (~> 1.16)
|
|
236
244
|
hnswlib (~> 0.9.0)
|
|
237
245
|
pg (~> 1.5)
|
|
238
246
|
pgvector (~> 0.2.2)
|
|
239
|
-
rake (~> 13.
|
|
247
|
+
rake (~> 13.2)
|
|
240
248
|
rest-client (~> 2.1)
|
|
241
|
-
rspec (~> 3.
|
|
242
|
-
rubocop (~> 1.
|
|
249
|
+
rspec (~> 3.13)
|
|
250
|
+
rubocop (~> 1.64)
|
|
243
251
|
rubocop-rake (~> 0.6.0)
|
|
244
|
-
rubocop-rspec (~> 2.
|
|
252
|
+
rubocop-rspec (~> 2.30)
|
|
245
253
|
sqlite3 (~> 1.7)
|
|
246
254
|
vcr (~> 6.2.0)
|
|
247
|
-
webmock (~> 3.
|
|
255
|
+
webmock (~> 3.23.1)
|
|
248
256
|
|
|
249
257
|
BUNDLED WITH
|
|
250
258
|
2.4.16
|
data/README.md
CHANGED
|
@@ -46,7 +46,7 @@ Or install it yourself as:
|
|
|
46
46
|
|
|
47
47
|
## Usage
|
|
48
48
|
|
|
49
|
-
We will be adding more examples soon, but here are a couple to get you started. First, you'll need to set up your environment variables for OpenAI and Google SERP (OPENAI_ACCESS_TOKEN, SERPAPI_API_KEY). If you prefer not to set these variables in your environment, you can pass them directly into the API.
|
|
49
|
+
We will be adding more examples soon, but here are a couple to get you started. First, you'll need to set up your environment variables for services like OpenAI, Anthropic, and Google SERP (OPENAI_ACCESS_TOKEN, ANTHROPIC_API_KEY,SERPAPI_API_KEY) etc. If you prefer not to set these variables in your environment, you can pass them directly into the API.
|
|
50
50
|
|
|
51
51
|
In the examples below, we added one Ruby gem to load the environment at the first line, but depending on what you want, you might not need this.
|
|
52
52
|
```ruby
|
data/boxcars.gemspec
CHANGED
|
@@ -37,7 +37,7 @@ Gem::Specification.new do |spec|
|
|
|
37
37
|
spec.add_dependency "hnswlib", "~> 0.8"
|
|
38
38
|
spec.add_dependency "nokogiri", "~> 1.16"
|
|
39
39
|
spec.add_dependency "pgvector", "~> 0.2"
|
|
40
|
-
spec.add_dependency "ruby-openai", ">= 4.2", "<
|
|
40
|
+
spec.add_dependency "ruby-openai", ">= 4.2", "< 8.0"
|
|
41
41
|
|
|
42
42
|
# For more information and examples about making a new gem, checkout our
|
|
43
43
|
# guide at: https://bundler.io/guides/creating_gem.html
|
|
@@ -99,10 +99,10 @@ module Boxcars
|
|
|
99
99
|
|
|
100
100
|
# check for dangerous code that is outside of ActiveRecord
|
|
101
101
|
def safe_to_run?(code)
|
|
102
|
-
bad_words = %w[commit drop_constraint drop_constraint! drop_extension drop_extension! drop_foreign_key drop_foreign_key!
|
|
103
|
-
drop_index drop_index! drop_join_table drop_join_table! drop_materialized_view drop_materialized_view!
|
|
104
|
-
drop_partition drop_partition! drop_schema drop_schema! drop_table drop_table! drop_trigger drop_trigger!
|
|
105
|
-
drop_view drop_view! eval instance_eval send system execute reset revoke rollback truncate
|
|
102
|
+
bad_words = %w[commit drop_constraint drop_constraint! drop_extension drop_extension! drop_foreign_key drop_foreign_key!
|
|
103
|
+
drop_index drop_index! drop_join_table drop_join_table! drop_materialized_view drop_materialized_view!
|
|
104
|
+
drop_partition drop_partition! drop_schema drop_schema! drop_table drop_table! drop_trigger drop_trigger!
|
|
105
|
+
drop_view drop_view! eval instance_eval send system execute reset revoke rollback truncate
|
|
106
106
|
encrypted_password].freeze
|
|
107
107
|
without_strings = code.gsub(/('([^'\\]*(\\.[^'\\]*)*)'|"([^"\\]*(\\.[^"\\]*)*"))/, 'XX')
|
|
108
108
|
|
|
@@ -25,8 +25,8 @@ module Boxcars
|
|
|
25
25
|
def run(question)
|
|
26
26
|
search = ::GoogleSearch.new(q: question)
|
|
27
27
|
rv = find_answer(search.get_hash)
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
Boxcars.info "Question: #{question}"
|
|
29
|
+
Boxcars.info "Answer: #{rv}"
|
|
30
30
|
rv
|
|
31
31
|
end
|
|
32
32
|
|
|
@@ -11,7 +11,7 @@ module Boxcars
|
|
|
11
11
|
# :name, :description, :prompt, :top_k, :stop, and :engine
|
|
12
12
|
def initialize(connection: nil, tables: nil, except_tables: nil, **kwargs)
|
|
13
13
|
connection ||= ::ActiveRecord::Base.connection
|
|
14
|
-
super
|
|
14
|
+
super
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
private
|
|
@@ -10,7 +10,7 @@ module Boxcars
|
|
|
10
10
|
# @param kwargs [Hash] Any other keyword arguments to pass to the parent class. This can include
|
|
11
11
|
# :name, :description, :prompt, :top_k, :stop, and :engine
|
|
12
12
|
def initialize(connection: nil, tables: nil, except_tables: nil, **kwargs)
|
|
13
|
-
super
|
|
13
|
+
super
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
private
|
|
@@ -10,7 +10,7 @@ module Boxcars
|
|
|
10
10
|
# @param name [String] The name of the boxcar. Defaults to classname.
|
|
11
11
|
# @param description [String] A description of the boxcar. Defaults to SERPDESC.
|
|
12
12
|
def initialize(name: "FetchURL", description: DESC)
|
|
13
|
-
super
|
|
13
|
+
super
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
# Get text from a url.
|
|
@@ -13,7 +13,7 @@ module Boxcars
|
|
|
13
13
|
# @param description [String] A description of the boxcar. Defaults to SERPDESC.
|
|
14
14
|
# @param serpapi_api_key [String] The API key to use for the SerpAPI. Defaults to Boxcars.configuration.serpapi_api_key.
|
|
15
15
|
def initialize(name: "Wikipedia", description: WDESC)
|
|
16
|
-
super
|
|
16
|
+
super
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
# Get an answer from Google using the SerpAPI.
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Boxcars - a framework for running a series of tools to get an answer to a question.
|
|
4
|
+
module Boxcars
|
|
5
|
+
# A engine that uses Cohere's API.
|
|
6
|
+
class Cohere < Engine
|
|
7
|
+
attr_reader :prompts, :llm_params, :model_kwargs, :batch_size
|
|
8
|
+
|
|
9
|
+
# The default parameters to use when asking the engine.
|
|
10
|
+
DEFAULT_PARAMS = {
|
|
11
|
+
model: "command-r-plus",
|
|
12
|
+
max_tokens: 4000,
|
|
13
|
+
max_input_tokens: 1000,
|
|
14
|
+
temperature: 0.2
|
|
15
|
+
}.freeze
|
|
16
|
+
|
|
17
|
+
# the default name of the engine
|
|
18
|
+
DEFAULT_NAME = "Cohere engine"
|
|
19
|
+
# the default description of the engine
|
|
20
|
+
DEFAULT_DESCRIPTION = "useful for when you need to use Cohere AI to answer questions. " \
|
|
21
|
+
"You should ask targeted questions"
|
|
22
|
+
|
|
23
|
+
# A engine is the driver for a single tool to run.
|
|
24
|
+
# @param name [String] The name of the engine. Defaults to "OpenAI engine".
|
|
25
|
+
# @param description [String] A description of the engine. Defaults to:
|
|
26
|
+
# useful for when you need to use AI to answer questions. You should ask targeted questions".
|
|
27
|
+
# @param prompts [Array<String>] The prompts to use when asking the engine. Defaults to [].
|
|
28
|
+
def initialize(name: DEFAULT_NAME, description: DEFAULT_DESCRIPTION, prompts: [], **kwargs)
|
|
29
|
+
@llm_params = DEFAULT_PARAMS.merge(kwargs)
|
|
30
|
+
@prompts = prompts
|
|
31
|
+
@batch_size = 20
|
|
32
|
+
super(description: description, name: name)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def conversation_model?(_model)
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def chat(params, cohere_api_key)
|
|
40
|
+
raise Boxcars::ConfigurationError('Cohere API key not set') if cohere_api_key.blank?
|
|
41
|
+
|
|
42
|
+
# Define the API endpoint and parameters
|
|
43
|
+
api_endpoint = 'https://api.cohere.ai/v1/chat'
|
|
44
|
+
|
|
45
|
+
connection = Faraday.new(api_endpoint) do |faraday|
|
|
46
|
+
faraday.request :url_encoded
|
|
47
|
+
faraday.headers['Authorization'] = "Bearer #{cohere_api_key}"
|
|
48
|
+
faraday.headers['Content-Type'] = 'application/json'
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Make the API call
|
|
52
|
+
response = connection.post { |req| req.body = params.to_json }
|
|
53
|
+
|
|
54
|
+
# response_data = JSON.parse(response.body, symbolize_names: true)
|
|
55
|
+
# response_data[:text]
|
|
56
|
+
JSON.parse(response.body, symbolize_names: true)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Get an answer from the engine.
|
|
60
|
+
# @param prompt [String] The prompt to use when asking the engine.
|
|
61
|
+
# @param cohere_api_key [String] Optional api key to use when asking the engine.
|
|
62
|
+
# Defaults to Boxcars.configuration.cohere_api_key.
|
|
63
|
+
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
|
64
|
+
def client(prompt:, inputs: {}, **kwargs)
|
|
65
|
+
api_key = Boxcars.configuration.cohere_api_key(**kwargs)
|
|
66
|
+
params = prompt.as_prompt(inputs: inputs, prefixes: default_prefixes, show_roles: true).merge(llm_params.merge(kwargs))
|
|
67
|
+
params[:message] = params.delete(:prompt)
|
|
68
|
+
params[:stop_sequences] = params.delete(:stop) if params.key?(:stop)
|
|
69
|
+
Boxcars.debug("Prompt after formatting:#{params[:message]}", :cyan) if Boxcars.configuration.log_prompts
|
|
70
|
+
chat(params, api_key)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# get an answer from the engine for a question.
|
|
74
|
+
# @param question [String] The question to ask the engine.
|
|
75
|
+
# @param kwargs [Hash] Additional parameters to pass to the engine if wanted.
|
|
76
|
+
def run(question, **kwargs)
|
|
77
|
+
prompt = Prompt.new(template: question)
|
|
78
|
+
response = client(prompt: prompt, **kwargs)
|
|
79
|
+
|
|
80
|
+
raise Error, "Cohere: No response from API" unless response
|
|
81
|
+
raise Error, "Cohere: #{response[:error]}" if response[:error]
|
|
82
|
+
|
|
83
|
+
answer = response[:text]
|
|
84
|
+
Boxcars.debug(response, :yellow)
|
|
85
|
+
answer
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Get the default parameters for the engine.
|
|
89
|
+
def default_params
|
|
90
|
+
llm_params
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Get generation informaton
|
|
94
|
+
# @param sub_choices [Array<Hash>] The choices to get generation info for.
|
|
95
|
+
# @return [Array<Generation>] The generation information.
|
|
96
|
+
def generation_info(sub_choices)
|
|
97
|
+
sub_choices.map do |choice|
|
|
98
|
+
Generation.new(
|
|
99
|
+
text: choice["completion"],
|
|
100
|
+
generation_info: {
|
|
101
|
+
finish_reason: choice.fetch("stop_reason", nil),
|
|
102
|
+
logprobs: choice.fetch("logprobs", nil)
|
|
103
|
+
}
|
|
104
|
+
)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# make sure we got a valid response
|
|
109
|
+
# @param response [Hash] The response to check.
|
|
110
|
+
# @param must_haves [Array<String>] The keys that must be in the response. Defaults to %w[choices].
|
|
111
|
+
# @raise [KeyError] if there is an issue with the access token.
|
|
112
|
+
# @raise [ValueError] if the response is not valid.
|
|
113
|
+
def check_response(response, must_haves: %w[completion])
|
|
114
|
+
if response['error']
|
|
115
|
+
code = response.dig('error', 'code')
|
|
116
|
+
msg = response.dig('error', 'message') || 'unknown error'
|
|
117
|
+
raise KeyError, "ANTHOPIC_API_KEY not valid" if code == 'invalid_api_key'
|
|
118
|
+
|
|
119
|
+
raise ValueError, "Cohere error: #{msg}"
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
must_haves.each do |key|
|
|
123
|
+
raise ValueError, "Expecting key #{key} in response" unless response.key?(key)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Call out to OpenAI's endpoint with k unique prompts.
|
|
128
|
+
# @param prompts [Array<String>] The prompts to pass into the model.
|
|
129
|
+
# @param inputs [Array<String>] The inputs to subsitite into the prompt.
|
|
130
|
+
# @param stop [Array<String>] Optional list of stop words to use when generating.
|
|
131
|
+
# @return [EngineResult] The full engine output.
|
|
132
|
+
def generate(prompts:, stop: nil)
|
|
133
|
+
params = {}
|
|
134
|
+
params[:stop] = stop if stop
|
|
135
|
+
choices = []
|
|
136
|
+
# Get the token usage from the response.
|
|
137
|
+
# Includes prompt, completion, and total tokens used.
|
|
138
|
+
prompts.each_slice(batch_size) do |sub_prompts|
|
|
139
|
+
sub_prompts.each do |sprompts, inputs|
|
|
140
|
+
response = client(prompt: sprompts, inputs: inputs, **params)
|
|
141
|
+
check_response(response)
|
|
142
|
+
choices << response
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
n = params.fetch(:n, 1)
|
|
147
|
+
generations = []
|
|
148
|
+
prompts.each_with_index do |_prompt, i|
|
|
149
|
+
sub_choices = choices[i * n, (i + 1) * n]
|
|
150
|
+
generations.push(generation_info(sub_choices))
|
|
151
|
+
end
|
|
152
|
+
EngineResult.new(generations: generations, engine_output: { token_usage: {} })
|
|
153
|
+
end
|
|
154
|
+
# rubocop:enable Metrics/AbcSize
|
|
155
|
+
|
|
156
|
+
# the engine type
|
|
157
|
+
def engine_type
|
|
158
|
+
"claude"
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# calculate the number of tokens used
|
|
162
|
+
def get_num_tokens(text:)
|
|
163
|
+
text.split.length # TODO: hook up to token counting gem
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# lookup the context size for a model by name
|
|
167
|
+
# @param modelname [String] The name of the model to lookup.
|
|
168
|
+
def modelname_to_contextsize(_modelname)
|
|
169
|
+
100000
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Calculate the maximum number of tokens possible to generate for a prompt.
|
|
173
|
+
# @param prompt_text [String] The prompt text to use.
|
|
174
|
+
# @return [Integer] the number of tokens possible to generate.
|
|
175
|
+
def max_tokens_for_prompt(prompt_text)
|
|
176
|
+
num_tokens = get_num_tokens(prompt_text)
|
|
177
|
+
|
|
178
|
+
# get max context size for model by name
|
|
179
|
+
max_size = modelname_to_contextsize(model_name)
|
|
180
|
+
max_size - num_tokens
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def default_prefixes
|
|
184
|
+
{ system: "SYSTEM: ", user: "USER: ", assistant: "CHATBOT: ", history: :history }
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
data/lib/boxcars/engine.rb
CHANGED
data/lib/boxcars/vector_store.rb
CHANGED
|
@@ -59,12 +59,11 @@ module Boxcars
|
|
|
59
59
|
end
|
|
60
60
|
|
|
61
61
|
def load_data_files(training_data_path)
|
|
62
|
-
data = []
|
|
63
62
|
files = Dir.glob(training_data_path)
|
|
64
63
|
raise_error "No files found at #{training_data_path}" if files.empty?
|
|
65
64
|
|
|
66
|
-
files.
|
|
67
|
-
|
|
65
|
+
data = files.map do |file|
|
|
66
|
+
File.read(file)
|
|
68
67
|
end
|
|
69
68
|
puts "Added #{files.length} files to data. Splitting text into chunks..."
|
|
70
69
|
data
|
data/lib/boxcars/version.rb
CHANGED
data/lib/boxcars.rb
CHANGED
|
@@ -52,6 +52,11 @@ module Boxcars
|
|
|
52
52
|
key_lookup(:anthropic_api_key, kwargs)
|
|
53
53
|
end
|
|
54
54
|
|
|
55
|
+
# @return [String] The Anthropic API key either from arg or env.
|
|
56
|
+
def cohere_api_key(**kwargs)
|
|
57
|
+
key_lookup(:cohere_api_key, kwargs)
|
|
58
|
+
end
|
|
59
|
+
|
|
55
60
|
private
|
|
56
61
|
|
|
57
62
|
def check_key(key, val)
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: boxcars
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Francis Sullivan
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: exe
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2024-
|
|
12
|
+
date: 2024-06-14 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: anthropic
|
|
@@ -104,7 +104,7 @@ dependencies:
|
|
|
104
104
|
version: '4.2'
|
|
105
105
|
- - "<"
|
|
106
106
|
- !ruby/object:Gem::Version
|
|
107
|
-
version: '
|
|
107
|
+
version: '8.0'
|
|
108
108
|
type: :runtime
|
|
109
109
|
prerelease: false
|
|
110
110
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -114,7 +114,7 @@ dependencies:
|
|
|
114
114
|
version: '4.2'
|
|
115
115
|
- - "<"
|
|
116
116
|
- !ruby/object:Gem::Version
|
|
117
|
-
version: '
|
|
117
|
+
version: '8.0'
|
|
118
118
|
description: You simply set an OpenAI key, give a number of Boxcars to a Train, and
|
|
119
119
|
magic ensues when you run it.
|
|
120
120
|
email:
|
|
@@ -157,6 +157,7 @@ files:
|
|
|
157
157
|
- lib/boxcars/conversation_prompt.rb
|
|
158
158
|
- lib/boxcars/engine.rb
|
|
159
159
|
- lib/boxcars/engine/anthropic.rb
|
|
160
|
+
- lib/boxcars/engine/cohere.rb
|
|
160
161
|
- lib/boxcars/engine/engine_result.rb
|
|
161
162
|
- lib/boxcars/engine/gpt4all_eng.rb
|
|
162
163
|
- lib/boxcars/engine/openai.rb
|