openai.rb 0.0.0 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +27 -0
- data/.rubocop.yml +18 -0
- data/.ruby-version +1 -1
- data/Gemfile +9 -5
- data/Gemfile.lock +29 -24
- data/README.md +401 -0
- data/bin/console +9 -4
- data/lib/openai/api/cache.rb +137 -0
- data/lib/openai/api/client.rb +86 -0
- data/lib/openai/api/resource.rb +232 -0
- data/lib/openai/api/response.rb +384 -0
- data/lib/openai/api.rb +75 -0
- data/lib/openai/chat.rb +125 -0
- data/lib/openai/tokenizer.rb +50 -0
- data/lib/openai/util.rb +47 -0
- data/lib/openai/version.rb +1 -1
- data/lib/openai.rb +38 -357
- data/openai.gemspec +9 -3
- data/spec/data/sample_french.mp3 +0 -0
- data/spec/data/sample_image.png +0 -0
- data/spec/data/sample_image_mask.png +0 -0
- data/spec/shared/api_resource_context.rb +22 -0
- data/spec/spec_helper.rb +4 -0
- data/spec/unit/openai/api/audio_spec.rb +78 -0
- data/spec/unit/openai/api/cache_spec.rb +115 -0
- data/spec/unit/openai/api/chat_completions_spec.rb +130 -0
- data/spec/unit/openai/api/completions_spec.rb +125 -0
- data/spec/unit/openai/api/edits_spec.rb +40 -0
- data/spec/unit/openai/api/embeddings_spec.rb +45 -0
- data/spec/unit/openai/api/files_spec.rb +163 -0
- data/spec/unit/openai/api/fine_tunes_spec.rb +322 -0
- data/spec/unit/openai/api/images_spec.rb +137 -0
- data/spec/unit/openai/api/models_spec.rb +98 -0
- data/spec/unit/openai/api/moderations_spec.rb +63 -0
- data/spec/unit/openai/api/response_spec.rb +203 -0
- data/spec/unit/openai/chat_spec.rb +32 -0
- data/spec/unit/openai/tokenizer_spec.rb +45 -0
- data/spec/unit/openai_spec.rb +47 -736
- metadata +97 -7
- data/bin/codegen +0 -371
@@ -0,0 +1,137 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class OpenAI
|
4
|
+
class API
|
5
|
+
class Cache
|
6
|
+
include Concord.new(:client, :cache)
|
7
|
+
|
8
|
+
def get(route)
|
9
|
+
read_cache_or_apply(verb: :get, route: route) do
|
10
|
+
client.get(route)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Caching is a no-op for delete requests since caching a delete does
|
15
|
+
# not really make sense
|
16
|
+
def delete(...)
|
17
|
+
client.delete(...)
|
18
|
+
end
|
19
|
+
|
20
|
+
def post(route, **body)
|
21
|
+
read_cache_or_apply(verb: :post, route: route, body: body, format: :json) do
|
22
|
+
client.post(route, **body)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def post_form_multipart(route, **body)
|
27
|
+
read_cache_or_apply(verb: :post, route: route, body: body, format: :form) do
|
28
|
+
client.post_form_multipart(route, **body)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
def read_cache_or_apply(...)
|
35
|
+
target = cache_target(...).unique_id
|
36
|
+
|
37
|
+
if cache.cached?(target)
|
38
|
+
cache.read(target)
|
39
|
+
else
|
40
|
+
yield.tap do |result|
|
41
|
+
cache.write(target, result)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def cache_target(verb:, route:, body: nil, format: nil)
|
47
|
+
Target.new(
|
48
|
+
verb: verb,
|
49
|
+
api_key: client.api_key,
|
50
|
+
route: route,
|
51
|
+
body: body,
|
52
|
+
format: format
|
53
|
+
)
|
54
|
+
end
|
55
|
+
|
56
|
+
class Target
|
57
|
+
include Anima.new(:verb, :api_key, :route, :body, :format)
|
58
|
+
include Memoizable
|
59
|
+
|
60
|
+
def unique_id
|
61
|
+
serialized = JSON.dump(serialize_for_cache)
|
62
|
+
digest = Digest::SHA256.hexdigest(serialized)
|
63
|
+
bare_route = route.delete_prefix('/v1/')
|
64
|
+
prefix = "#{verb}_#{bare_route}".gsub('/', '_')
|
65
|
+
fingerprint = digest.slice(0, 8)
|
66
|
+
|
67
|
+
"#{prefix}_#{fingerprint}"
|
68
|
+
end
|
69
|
+
memoize :unique_id
|
70
|
+
|
71
|
+
def serialize_for_cache
|
72
|
+
data = to_h
|
73
|
+
if data[:body]
|
74
|
+
data[:body] = data[:body].transform_values do |value|
|
75
|
+
if value.instance_of?(HTTP::FormData::File)
|
76
|
+
Digest::SHA256.hexdigest(value.to_s)
|
77
|
+
else
|
78
|
+
value
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
data
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
class Strategy
|
87
|
+
include AbstractType
|
88
|
+
|
89
|
+
abstract_method :cached?
|
90
|
+
abstract_method :read
|
91
|
+
abstract_method :write
|
92
|
+
|
93
|
+
class Memory < self
|
94
|
+
include Concord.new(:cache)
|
95
|
+
|
96
|
+
def initialize
|
97
|
+
super({})
|
98
|
+
end
|
99
|
+
|
100
|
+
def cached?(target)
|
101
|
+
cache.key?(target)
|
102
|
+
end
|
103
|
+
|
104
|
+
def write(target, result)
|
105
|
+
cache[target] = result
|
106
|
+
end
|
107
|
+
|
108
|
+
def read(target)
|
109
|
+
cache.fetch(target)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
class FileSystem < self
|
114
|
+
include Concord.new(:cache_dir)
|
115
|
+
|
116
|
+
def cached?(target)
|
117
|
+
cache_file(target).file?
|
118
|
+
end
|
119
|
+
|
120
|
+
def write(target, result)
|
121
|
+
cache_file(target).write(result)
|
122
|
+
end
|
123
|
+
|
124
|
+
def read(target)
|
125
|
+
cache_file(target).read
|
126
|
+
end
|
127
|
+
|
128
|
+
private
|
129
|
+
|
130
|
+
def cache_file(target)
|
131
|
+
cache_dir.join("#{target}.json")
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class OpenAI
|
4
|
+
class API
|
5
|
+
class Client
|
6
|
+
include Concord.new(:api_key, :http)
|
7
|
+
|
8
|
+
public :api_key
|
9
|
+
|
10
|
+
HOST = Addressable::URI.parse('https://api.openai.com/v1')
|
11
|
+
|
12
|
+
def initialize(api_key, http: HTTP)
|
13
|
+
super(api_key, http)
|
14
|
+
end
|
15
|
+
|
16
|
+
def inspect
|
17
|
+
"#<#{self.class}>"
|
18
|
+
end
|
19
|
+
|
20
|
+
def get(route)
|
21
|
+
unwrap_response(json_http_client.get(url_for(route)))
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete(route)
|
25
|
+
unwrap_response(json_http_client.delete(url_for(route)))
|
26
|
+
end
|
27
|
+
|
28
|
+
def post(route, **body)
|
29
|
+
url = url_for(route)
|
30
|
+
if block_given?
|
31
|
+
json_http_client.persistent(url) do |connection|
|
32
|
+
response = connection.post(url, json: body)
|
33
|
+
|
34
|
+
# Data comes in as JSON frames like so:
|
35
|
+
#
|
36
|
+
# data: {"choices": [{"text": "He"}]}
|
37
|
+
# data: {"choices": [{"text": "llo, "}]}
|
38
|
+
# data: {"choices": [{"text": "Wor"}]}
|
39
|
+
# data: {"choices": [{"text": "ld!"}]}
|
40
|
+
# data: [DONE]
|
41
|
+
#
|
42
|
+
# (The actual frames are fully formed JSON objects just like a
|
43
|
+
# non-streamed response, the examples above are just for brevity)
|
44
|
+
response.body.each do |chunk|
|
45
|
+
chunk.split("\n\n").each do |part|
|
46
|
+
frame = part.delete_prefix('data: ').strip
|
47
|
+
|
48
|
+
yield(frame) unless frame == '[DONE]' || frame.empty?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
# Return nil since we aren't reconstructing what the API would have
|
54
|
+
# returned if we had not streamed the response
|
55
|
+
nil
|
56
|
+
else
|
57
|
+
unwrap_response(json_http_client.post(url, json: body))
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def post_form_multipart(route, **body)
|
62
|
+
unwrap_response(http_client.post(url_for(route), form: body))
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def url_for(route)
|
68
|
+
HOST.join(route).to_str
|
69
|
+
end
|
70
|
+
|
71
|
+
def unwrap_response(response)
|
72
|
+
raise API::Error.parse(response) unless response.status.success?
|
73
|
+
|
74
|
+
response.body.to_str
|
75
|
+
end
|
76
|
+
|
77
|
+
def json_http_client
|
78
|
+
http_client.headers('Content-Type' => 'application/json')
|
79
|
+
end
|
80
|
+
|
81
|
+
def http_client
|
82
|
+
http.headers('Authorization' => "Bearer #{api_key}")
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,232 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class OpenAI
|
4
|
+
class API
|
5
|
+
class Resource
|
6
|
+
include Concord.new(:client)
|
7
|
+
include AbstractType
|
8
|
+
|
9
|
+
private
|
10
|
+
|
11
|
+
def post(...)
|
12
|
+
client.post(...)
|
13
|
+
end
|
14
|
+
|
15
|
+
def post_form_multipart(...)
|
16
|
+
client.post_form_multipart(...)
|
17
|
+
end
|
18
|
+
|
19
|
+
def get(...)
|
20
|
+
client.get(...)
|
21
|
+
end
|
22
|
+
|
23
|
+
def form_file(path)
|
24
|
+
absolute_path = Pathname.new(path).expand_path.to_s
|
25
|
+
HTTP::FormData::File.new(absolute_path)
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_and_maybe_stream(
|
29
|
+
endpoint,
|
30
|
+
full_response_type:, stream: false,
|
31
|
+
chunk_response_type: full_response_type,
|
32
|
+
**kwargs
|
33
|
+
)
|
34
|
+
payload = kwargs.merge(stream: stream)
|
35
|
+
|
36
|
+
raise 'Streaming responses require a block' if stream && !block_given?
|
37
|
+
raise 'Non-streaming responses do not support blocks' if !stream && block_given?
|
38
|
+
|
39
|
+
if stream
|
40
|
+
post(endpoint, **payload) do |chunk|
|
41
|
+
yield(chunk_response_type.from_json(chunk))
|
42
|
+
end
|
43
|
+
|
44
|
+
nil
|
45
|
+
else
|
46
|
+
full_response_type.from_json(
|
47
|
+
post(endpoint, **payload)
|
48
|
+
)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
class Completion < Resource
|
54
|
+
def create(model:, **kwargs, &block)
|
55
|
+
create_and_maybe_stream(
|
56
|
+
'/v1/completions',
|
57
|
+
model: model,
|
58
|
+
full_response_type: Response::Completion,
|
59
|
+
**kwargs,
|
60
|
+
&block
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
class ChatCompletion < Resource
|
66
|
+
def create(model:, messages:, **kwargs, &block)
|
67
|
+
create_and_maybe_stream(
|
68
|
+
'/v1/chat/completions',
|
69
|
+
model: model,
|
70
|
+
messages: messages,
|
71
|
+
full_response_type: Response::ChatCompletion,
|
72
|
+
chunk_response_type: Response::ChatCompletionChunk,
|
73
|
+
**kwargs,
|
74
|
+
&block
|
75
|
+
)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
class Embedding < Resource
|
80
|
+
def create(model:, input:, **kwargs)
|
81
|
+
Response::Embedding.from_json(
|
82
|
+
post('/v1/embeddings', model: model, input: input, **kwargs)
|
83
|
+
)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
class Model < Resource
|
88
|
+
def list
|
89
|
+
Response::ListModel.from_json(get('/v1/models'))
|
90
|
+
end
|
91
|
+
|
92
|
+
def fetch(model_id)
|
93
|
+
Response::Model.from_json(
|
94
|
+
get("/v1/models/#{model_id}")
|
95
|
+
)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class Moderation < Resource
|
100
|
+
def create(input:, model:)
|
101
|
+
Response::Moderation.from_json(
|
102
|
+
post('/v1/moderations', input: input, model: model)
|
103
|
+
)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
class Edit < Resource
|
108
|
+
def create(model:, instruction:, **kwargs)
|
109
|
+
Response::Edit.from_json(
|
110
|
+
post('/v1/edits', model: model, instruction: instruction, **kwargs)
|
111
|
+
)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
class File < Resource
|
116
|
+
def create(file:, purpose:)
|
117
|
+
Response::File.from_json(
|
118
|
+
post_form_multipart('/v1/files', file: form_file(file), purpose: purpose)
|
119
|
+
)
|
120
|
+
end
|
121
|
+
|
122
|
+
def list
|
123
|
+
Response::FileList.from_json(
|
124
|
+
get('/v1/files')
|
125
|
+
)
|
126
|
+
end
|
127
|
+
|
128
|
+
def delete(file_id)
|
129
|
+
Response::File.from_json(
|
130
|
+
client.delete("/v1/files/#{file_id}")
|
131
|
+
)
|
132
|
+
end
|
133
|
+
|
134
|
+
def fetch(file_id)
|
135
|
+
Response::File.from_json(
|
136
|
+
get("/v1/files/#{file_id}")
|
137
|
+
)
|
138
|
+
end
|
139
|
+
|
140
|
+
def get_content(file_id)
|
141
|
+
get("/v1/files/#{file_id}/content")
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
class FineTune < Resource
|
146
|
+
def list
|
147
|
+
Response::FineTuneList.from_json(
|
148
|
+
get('/v1/fine-tunes')
|
149
|
+
)
|
150
|
+
end
|
151
|
+
|
152
|
+
def create(training_file:, **kwargs)
|
153
|
+
Response::FineTune.from_json(
|
154
|
+
post('/v1/fine-tunes', training_file: training_file, **kwargs)
|
155
|
+
)
|
156
|
+
end
|
157
|
+
|
158
|
+
def fetch(fine_tune_id)
|
159
|
+
Response::FineTune.from_json(
|
160
|
+
get("/v1/fine-tunes/#{fine_tune_id}")
|
161
|
+
)
|
162
|
+
end
|
163
|
+
|
164
|
+
def cancel(fine_tune_id)
|
165
|
+
Response::FineTune.from_json(
|
166
|
+
post("/v1/fine-tunes/#{fine_tune_id}/cancel")
|
167
|
+
)
|
168
|
+
end
|
169
|
+
|
170
|
+
def list_events(fine_tune_id)
|
171
|
+
Response::FineTuneEventList.from_json(
|
172
|
+
get("/v1/fine-tunes/#{fine_tune_id}/events")
|
173
|
+
)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
class Image < Resource
|
178
|
+
def create(prompt:, **kwargs)
|
179
|
+
Response::ImageGeneration.from_json(
|
180
|
+
post('/v1/images/generations', prompt: prompt, **kwargs)
|
181
|
+
)
|
182
|
+
end
|
183
|
+
|
184
|
+
def create_variation(image:, **kwargs)
|
185
|
+
Response::ImageVariation.from_json(
|
186
|
+
post_form_multipart('/v1/images/variations', {
|
187
|
+
image: form_file(image),
|
188
|
+
**kwargs
|
189
|
+
})
|
190
|
+
)
|
191
|
+
end
|
192
|
+
|
193
|
+
def edit(image:, prompt:, mask: nil, **kwargs)
|
194
|
+
params = {
|
195
|
+
image: form_file(image),
|
196
|
+
prompt: prompt,
|
197
|
+
**kwargs
|
198
|
+
}
|
199
|
+
|
200
|
+
params[:mask] = form_file(mask) if mask
|
201
|
+
|
202
|
+
Response::ImageEdit.from_json(
|
203
|
+
post_form_multipart('/v1/images/edits', **params)
|
204
|
+
)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
class Audio < Resource
|
209
|
+
def transcribe(file:, model:, **kwargs)
|
210
|
+
Response::Transcription.from_json(
|
211
|
+
post_form_multipart(
|
212
|
+
'/v1/audio/transcriptions',
|
213
|
+
file: form_file(file),
|
214
|
+
model: model,
|
215
|
+
**kwargs
|
216
|
+
)
|
217
|
+
)
|
218
|
+
end
|
219
|
+
|
220
|
+
def translate(file:, model:, **kwargs)
|
221
|
+
Response::Transcription.from_json(
|
222
|
+
post_form_multipart(
|
223
|
+
'/v1/audio/translations',
|
224
|
+
file: form_file(file),
|
225
|
+
model: model,
|
226
|
+
**kwargs
|
227
|
+
)
|
228
|
+
)
|
229
|
+
end
|
230
|
+
end
|
231
|
+
end
|
232
|
+
end
|