x-uploads 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.yardopts +8 -0
- data/CHANGELOG.md +171 -0
- data/LICENSE.txt +21 -0
- data/README.md +98 -0
- data/lib/x/uploads/account.rb +100 -0
- data/lib/x/uploads/alt_text_failed.rb +81 -0
- data/lib/x/uploads/api.rb +310 -0
- data/lib/x/uploads/chunked_upload_failed.rb +89 -0
- data/lib/x/uploads/chunks.rb +325 -0
- data/lib/x/uploads/error.rb +25 -0
- data/lib/x/uploads/gif.rb +88 -0
- data/lib/x/uploads/invalid_media.rb +19 -0
- data/lib/x/uploads/invalid_media_type.rb +12 -0
- data/lib/x/uploads/json_classes.rb +14 -0
- data/lib/x/uploads/media_processing_check_failed.rb +85 -0
- data/lib/x/uploads/media_processing_failed.rb +43 -0
- data/lib/x/uploads/media_processing_timeout.rb +55 -0
- data/lib/x/uploads/media_upload.rb +565 -0
- data/lib/x/uploads/metadata.rb +97 -0
- data/lib/x/uploads/missing_media_data.rb +47 -0
- data/lib/x/uploads/multipart.rb +63 -0
- data/lib/x/uploads/signature.rb +104 -0
- data/lib/x/uploads/source.rb +440 -0
- data/lib/x/uploads/uploaded_media.rb +369 -0
- data/lib/x/uploads/utils.rb +330 -0
- data/lib/x/uploads/validator.rb +391 -0
- data/lib/x/uploads/version.rb +24 -0
- data/lib/x/uploads.rb +18 -0
- data/sig/manifest.yaml +6 -0
- data/sig/x-uploads.rbs +139 -0
- metadata +98 -0
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
require_relative "chunked_upload_failed"
|
|
5
|
+
require_relative "json_classes"
|
|
6
|
+
require_relative "missing_media_data"
|
|
7
|
+
require_relative "multipart"
|
|
8
|
+
require_relative "uploaded_media"
|
|
9
|
+
require_relative "utils"
|
|
10
|
+
require_relative "validator"
|
|
11
|
+
|
|
12
|
+
module X
|
|
13
|
+
module Uploads
|
|
14
|
+
# Uploads a file in the chunks the X API requires for video and subtitles
|
|
15
|
+
#
|
|
16
|
+
# Internal to x-uploads: X::Uploads::MediaUpload calls it rather than mix its methods into itself, so a class that
|
|
17
|
+
# includes X::Uploads::MediaUpload gains none of them.
|
|
18
|
+
#
|
|
19
|
+
# @api private
|
|
20
|
+
module Chunks
|
|
21
|
+
extend self
|
|
22
|
+
|
|
23
|
+
# The message of the error raised for an initialize response that holds no media to append the chunks to
|
|
24
|
+
NO_MEDIA = "The response that initializes the upload holds no media to append the chunks to"
|
|
25
|
+
# The message of the error raised for a chunk that reads fewer bytes than the media held when it was measured
|
|
26
|
+
CHANGED = "%s held %d bytes when the upload was initialized, but chunk %d read %d of the %d it began with at " \
|
|
27
|
+
"byte %d: the media changed while it was uploaded"
|
|
28
|
+
# The fiber-local key x-core reads the guard it runs the save_tokens of a refresh inside from
|
|
29
|
+
REPORT_GUARD = :x_core_refresh_report_guard
|
|
30
|
+
private_constant :NO_MEDIA, :CHANGED, :REPORT_GUARD
|
|
31
|
+
|
|
32
|
+
# Raised in a worker to stop it, which the worker rescues to end at once
|
|
33
|
+
#
|
|
34
|
+
# It is not a StandardError, as Timeout::ExitException is not, so that a callback the client runs in the worker,
|
|
35
|
+
# as an on_response or a load_tokens that rescues the errors of its own store, cannot rescue it and let the upload
|
|
36
|
+
# go on after the call has ended, as Thread#kill, which cannot be rescued, did not.
|
|
37
|
+
class Stopped < Exception; end # rubocop:disable Lint/InheritException
|
|
38
|
+
private_constant :Stopped
|
|
39
|
+
|
|
40
|
+
# Upload media in chunks: initialize, append each chunk, and finalize
|
|
41
|
+
#
|
|
42
|
+
# MediaUpload.upload and MediaUpload.chunked_upload both upload with it, once each has checked its other
|
|
43
|
+
# arguments, so that upload calls no method a class that includes MediaUpload could define in place of its own.
|
|
44
|
+
# The chunk size is checked against the segments the API numbers before the upload is initialized.
|
|
45
|
+
#
|
|
46
|
+
# @api private
|
|
47
|
+
# @param client [Client] the X API client
|
|
48
|
+
# @param source [Source] the media
|
|
49
|
+
# @param media_type [String] the MIME type
|
|
50
|
+
# @param media_category [String] the media category
|
|
51
|
+
# @param chunk_size [Integer, nil] the chunk size in bytes, or nil for one derived from the size of the media
|
|
52
|
+
# @param concurrency [Integer] the number of chunks uploaded at once
|
|
53
|
+
# @param shared [Boolean, nil] whether the media is shared, or nil to send none
|
|
54
|
+
# @param additional_owners [Array<Integer, String>, nil] the identifiers of the users who may use the media, or
|
|
55
|
+
# nil for none
|
|
56
|
+
# @return [UploadedMedia] the uploaded media, as the response that finalizes the upload describes it
|
|
57
|
+
# @raise [ArgumentError] if the chunk size would need more segments than the API numbers
|
|
58
|
+
# @raise [MissingMediaData] if the response that initializes the upload holds no media to append the chunks to
|
|
59
|
+
# @raise [ChunkedUploadFailed] if the upload is initialized, but a chunk cannot be appended, or it cannot be
|
|
60
|
+
# finalized, or the response that finalizes it holds no media, with the media it initialized
|
|
61
|
+
# @example Upload a video in chunks
|
|
62
|
+
# Uploads::Chunks.upload(client:, source:, media_type: "video/mp4", media_category: "tweet_video",
|
|
63
|
+
# chunk_size: 4_194_304, concurrency: 4)
|
|
64
|
+
def upload(client:, source:, media_type:, media_category:, chunk_size:, concurrency:, shared:, additional_owners:)
|
|
65
|
+
chunk_size = Validator.validate_segments!(source, chunk_size)
|
|
66
|
+
media = init(client:, source:, media_type:, media_category:, shared:, additional_owners:)
|
|
67
|
+
ChunkedUploadFailed.__send__(:keeping, media) { complete(client:, source:, chunk_size:, media:, concurrency:) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Initialize a chunked upload
|
|
71
|
+
#
|
|
72
|
+
# The chunks that follow are appended to the media this returns, so a response that holds none, whether it has
|
|
73
|
+
# no body at all, a body without data, or data that names no media, raises here rather than leave the upload
|
|
74
|
+
# to fail on the first chunk, once the media it uploaded had been billed.
|
|
75
|
+
#
|
|
76
|
+
# @api private
|
|
77
|
+
# @param client [Client] the X API client
|
|
78
|
+
# @param source [Source] the media
|
|
79
|
+
# @param media_type [String] the MIME type
|
|
80
|
+
# @param media_category [String] the media category
|
|
81
|
+
# @param shared [Boolean, nil] whether the media is shared, or nil to send none
|
|
82
|
+
# @param additional_owners [Array<Integer, String>, nil] the identifiers of the users who may use the media, or
|
|
83
|
+
# nil for none
|
|
84
|
+
# @return [Hash] the media the chunks are appended to
|
|
85
|
+
# @raise [MissingMediaData] if the response holds no media to append the chunks to
|
|
86
|
+
# @example Initialize the upload of a video
|
|
87
|
+
# Uploads::Chunks.init(client:, source:, media_type: "video/mp4", media_category: "tweet_video")
|
|
88
|
+
def init(client:, source:, media_type:, media_category:, shared: nil, additional_owners: nil)
|
|
89
|
+
body = {media_type:, media_category:, total_bytes: source.size, shared:, additional_owners: additional_owners&.map(&:to_s)}.compact
|
|
90
|
+
response = client.post("media/upload/initialize", body, **JSON_CLASSES)
|
|
91
|
+
body = Utils.body_of(response)
|
|
92
|
+
media = body["data"]
|
|
93
|
+
raise MissingMediaData.new(NO_MEDIA, problems: Problem.all_from(body)) unless UploadedMedia.__send__(:documented?, media)
|
|
94
|
+
|
|
95
|
+
media
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Append the chunks of a file to a chunked upload, a few at a time
|
|
99
|
+
#
|
|
100
|
+
# Each worker reads its chunk from the media as it uploads it, so no more than concurrency chunks are held
|
|
101
|
+
# in memory at once. A chunk that fails stops the chunks not yet begun, and once the chunks already begun
|
|
102
|
+
# have finished, the first error is raised, unless save_tokens raised for the tokens of a refresh a chunk made,
|
|
103
|
+
# whose TokenReportFailed is raised in its place, whichever chunk failed first, since no other error holds the
|
|
104
|
+
# tokens. An exception raised in the caller while it waits, such as a timeout or an interrupt, stops every
|
|
105
|
+
# chunk, so that no thread goes on uploading once the caller has gone.
|
|
106
|
+
#
|
|
107
|
+
# @api private
|
|
108
|
+
# @param client [Client] the X API client
|
|
109
|
+
# @param source [Source] the media
|
|
110
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
111
|
+
# @param media [Hash] the media object
|
|
112
|
+
# @param boundary [String] the multipart boundary
|
|
113
|
+
# @param concurrency [Integer] the number of chunks uploaded at once
|
|
114
|
+
# @return [void]
|
|
115
|
+
# @raise [TokenReportFailed] if save_tokens raises for the tokens of a refresh a chunk made, with the tokens
|
|
116
|
+
# @raise [StandardError] if a chunk fails otherwise, the error of the first that failed
|
|
117
|
+
# @example Append the chunks of a video
|
|
118
|
+
# Uploads::Chunks.append(client:, source:, chunk_size: 4_194_304, media:, boundary:, concurrency: 4)
|
|
119
|
+
def append(client:, source:, chunk_size:, media:, boundary:, concurrency:)
|
|
120
|
+
queue = chunk_queue(source, chunk_size)
|
|
121
|
+
errors = Queue.new
|
|
122
|
+
media_id = media.fetch("id")
|
|
123
|
+
await([concurrency, queue.size].min, queue) { append_worker(queue, errors, client:, source:, chunk_size:, media_id:, boundary:) }
|
|
124
|
+
error = failure(errors)
|
|
125
|
+
raise error if error
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Append the chunks of a file to a chunked upload and finalize it
|
|
129
|
+
#
|
|
130
|
+
# @api private
|
|
131
|
+
# @param client [Client] the X API client
|
|
132
|
+
# @param source [Source] the media
|
|
133
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
134
|
+
# @param media [Hash] the media the upload initialized
|
|
135
|
+
# @param concurrency [Integer] the number of chunks uploaded at once
|
|
136
|
+
# @return [UploadedMedia] the uploaded media, as the response that finalizes the upload describes it
|
|
137
|
+
# @raise [MissingMediaData] if the response that finalizes the upload holds no media or carries no body at all
|
|
138
|
+
# @example Upload the chunks of a video and finalize it
|
|
139
|
+
# Uploads::Chunks.complete(client:, source:, chunk_size: 4_194_304, media:, concurrency: 4)
|
|
140
|
+
def complete(client:, source:, chunk_size:, media:, concurrency:)
|
|
141
|
+
append(client:, source:, chunk_size:, media:, boundary: SecureRandom.hex, concurrency:)
|
|
142
|
+
UploadedMedia.new(Utils.media_data(finalize(client:, media:), "that finalizes the upload"))
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Finalize a chunked upload, once its chunks are appended
|
|
146
|
+
#
|
|
147
|
+
# It is sent again after a server or network error, as a chunk is, rather than lose an upload whose every
|
|
148
|
+
# chunk was appended to a failure that may pass: the media can be finalized only by the upload that
|
|
149
|
+
# initialized it, which knows its identifier. A finalize the API acted on, but whose answer never arrived, may
|
|
150
|
+
# be refused when it is sent again, which raises as the failure it follows would have.
|
|
151
|
+
#
|
|
152
|
+
# @api private
|
|
153
|
+
# @param client [Client] the X API client
|
|
154
|
+
# @param media [Hash] the media the chunks were appended to
|
|
155
|
+
# @return [Hash, nil] the parsed response, or nil for a response that carries no body at all
|
|
156
|
+
# @example Finalize the upload of a video
|
|
157
|
+
# Uploads::Chunks.finalize(client:, media: {"id" => "1880028106020515840"})
|
|
158
|
+
def finalize(client:, media:)
|
|
159
|
+
Utils.sending_again(client) { client.post("media/upload/#{media.fetch("id")}/finalize", **JSON_CLASSES) }
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private
|
|
163
|
+
|
|
164
|
+
# Start the workers and wait for them, stopping them if the wait is cut short
|
|
165
|
+
#
|
|
166
|
+
# Every worker has finished when the wait ends on its own, so there is nothing left to stop. When an exception is
|
|
167
|
+
# raised in the waiting thread, the workers are stopped, which closes the connection of a request under way, and
|
|
168
|
+
# waited for, so that none outlives the call. A worker that is storing the tokens of a refresh it made finishes
|
|
169
|
+
# first, so that its save_tokens is not cut short, leaving the store with a refresh token X no longer accepts,
|
|
170
|
+
# while a timeout of its own still ends it. The workers are started inside the wait, so that one that cannot be
|
|
171
|
+
# started, as a thread the system refuses raises ThreadError, or an exception raised in the caller while they are
|
|
172
|
+
# started, stops those started before it as well.
|
|
173
|
+
#
|
|
174
|
+
# Each worker is started, and the workers are stopped, holding back an exception raised in the caller, which
|
|
175
|
+
# would otherwise leave a worker it struck as it was started, or as the others were stopped, uploading every
|
|
176
|
+
# chunk left. An exception a trap raises is not held back, so the chunks not yet begun are emptied first, and a
|
|
177
|
+
# worker it leaves unstopped ends once the chunk it began is uploaded. A worker reading a chunk from the media
|
|
178
|
+
# finishes reading it first, so that the file it reads is closed.
|
|
179
|
+
#
|
|
180
|
+
# @api private
|
|
181
|
+
# @param count [Integer] the number of workers to start
|
|
182
|
+
# @param queue [Thread::Queue] the index and offset of each chunk not yet begun
|
|
183
|
+
# @yieldreturn [Thread] a worker it started, a thread that uploads chunks
|
|
184
|
+
# @return [void]
|
|
185
|
+
def await(count, queue)
|
|
186
|
+
workers = [] #: Array[Thread]
|
|
187
|
+
begin
|
|
188
|
+
count.times { Thread.handle_interrupt(Object => :never) { workers << yield } }
|
|
189
|
+
workers.each(&:join)
|
|
190
|
+
ensure
|
|
191
|
+
Thread.handle_interrupt(Object => :never) { stop(workers, queue) }
|
|
192
|
+
workers.each(&:join)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Empty the chunks not yet begun, then stop the workers
|
|
197
|
+
# @api private
|
|
198
|
+
# @param workers [Array<Thread>] the workers
|
|
199
|
+
# @param queue [Thread::Queue] the index and offset of each chunk not yet begun
|
|
200
|
+
# @return [void]
|
|
201
|
+
def stop(workers, queue)
|
|
202
|
+
queue.clear
|
|
203
|
+
workers.each { |worker| worker.raise(Stopped) }
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# The error the chunks that failed raise, of those they recorded
|
|
207
|
+
#
|
|
208
|
+
# It is that of the first chunk that failed, unless save_tokens raised for the tokens of a refresh a chunk made:
|
|
209
|
+
# a TokenReportFailed is raised in place of any other error, since it alone holds tokens the store has yet to
|
|
210
|
+
# hold, and the last of them, if more than one refresh was made, since the refresh token of an earlier one is
|
|
211
|
+
# spent.
|
|
212
|
+
#
|
|
213
|
+
# @api private
|
|
214
|
+
# @param errors [Thread::Queue] the errors of failed chunks, in the order they failed
|
|
215
|
+
# @return [StandardError, nil] the error to raise, or nil if no chunk failed
|
|
216
|
+
def failure(errors)
|
|
217
|
+
failures = Array.new(errors.size) { errors.deq }
|
|
218
|
+
failures.reverse_each.find { |error| TokenReportFailed === error } || failures.first
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# A closed queue of the index and byte offset of each chunk of the media, in order
|
|
222
|
+
# @api private
|
|
223
|
+
# @param source [Source] the media
|
|
224
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
225
|
+
# @return [Thread::Queue] the queue
|
|
226
|
+
def chunk_queue(source, chunk_size)
|
|
227
|
+
queue = Queue.new
|
|
228
|
+
(0...source.size).step(chunk_size).each_with_index { |offset, index| queue << [index, offset] }
|
|
229
|
+
queue.close
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# Start a thread that uploads chunks from a queue, emptying it if a chunk fails
|
|
233
|
+
#
|
|
234
|
+
# The thread is started holding every exception back, as a new thread holds back what the thread that starts it
|
|
235
|
+
# does, and takes them once it runs where it rescues Stopped, so that one stopped before it begins ends there
|
|
236
|
+
# rather than with Stopped, and a timeout of its own save_tokens still ends it.
|
|
237
|
+
#
|
|
238
|
+
# @api private
|
|
239
|
+
# @param queue [Thread::Queue] the index and offset of each chunk not yet begun
|
|
240
|
+
# @param errors [Thread::Queue] the errors of failed chunks, in the order they failed
|
|
241
|
+
# @param client [Client] the X API client
|
|
242
|
+
# @param source [Source] the media
|
|
243
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
244
|
+
# @param media_id [String] the media ID
|
|
245
|
+
# @param boundary [String] the multipart boundary
|
|
246
|
+
# @return [Thread] the thread
|
|
247
|
+
def append_worker(queue, errors, client:, source:, chunk_size:, media_id:, boundary:)
|
|
248
|
+
Thread.new do
|
|
249
|
+
Thread.handle_interrupt(Object => :immediate) { append_chunks(queue, errors, client:, source:, chunk_size:, media_id:, boundary:) }
|
|
250
|
+
rescue Stopped
|
|
251
|
+
# A worker stopped before or between its requests ends as one stopped mid-request does
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Upload chunks from a queue until it is empty, emptying it if a chunk fails
|
|
256
|
+
# @api private
|
|
257
|
+
# @param queue [Thread::Queue] the index and offset of each chunk not yet begun
|
|
258
|
+
# @param errors [Thread::Queue] the errors of failed chunks, in the order they failed
|
|
259
|
+
# @param client [Client] the X API client
|
|
260
|
+
# @param source [Source] the media
|
|
261
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
262
|
+
# @param media_id [String] the media ID
|
|
263
|
+
# @param boundary [String] the multipart boundary
|
|
264
|
+
# @return [void]
|
|
265
|
+
def append_chunks(queue, errors, client:, source:, chunk_size:, media_id:, boundary:)
|
|
266
|
+
Thread.current[REPORT_GUARD] = method(:saving)
|
|
267
|
+
while (index, offset = queue.deq)
|
|
268
|
+
upload_body = Multipart.body("media", chunk(source, chunk_size, index, offset), boundary:, segment_index: index)
|
|
269
|
+
upload_chunk(client:, media_id:, upload_body:, headers: Multipart.headers(boundary))
|
|
270
|
+
end
|
|
271
|
+
rescue => e
|
|
272
|
+
errors << e
|
|
273
|
+
queue.clear
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# Run the save_tokens of a worker, holding a stop back until it returns
|
|
277
|
+
# @api private
|
|
278
|
+
# @yield passes the tokens of the refresh to save_tokens
|
|
279
|
+
# @return [Object] what the block returns
|
|
280
|
+
def saving(&) = Thread.handle_interrupt(Stopped => :never, &)
|
|
281
|
+
|
|
282
|
+
# Read a chunk of the media, of the bytes the upload declared it holds
|
|
283
|
+
#
|
|
284
|
+
# The size of the media is read once, so every chunk is read within the bytes the upload was initialized with,
|
|
285
|
+
# and a file that grows while it is uploaded appends none of what it grew by. A file that shrinks cannot fill
|
|
286
|
+
# the chunks it was measured for, so a chunk that reads short raises, rather than append fewer bytes than the
|
|
287
|
+
# upload declared.
|
|
288
|
+
#
|
|
289
|
+
# A stop of the worker is held back until the chunk is read. Ruby raises an exception raised in a thread once
|
|
290
|
+
# the file is opened, before anything holds the handle to close it, so a worker stopped as it opened the file
|
|
291
|
+
# would leave it open for as long as the process ran, and a file left open cannot be deleted on Windows.
|
|
292
|
+
#
|
|
293
|
+
# @api private
|
|
294
|
+
# @param source [Source] the media
|
|
295
|
+
# @param chunk_size [Integer] the chunk size in bytes
|
|
296
|
+
# @param index [Integer] the index of the chunk
|
|
297
|
+
# @param offset [Integer] the byte the chunk begins at
|
|
298
|
+
# @return [String] the bytes of the chunk
|
|
299
|
+
# @raise [EOFError] if the media holds fewer bytes than it did when it was measured
|
|
300
|
+
def chunk(source, chunk_size, index, offset)
|
|
301
|
+
length = [chunk_size, source.size - offset].min
|
|
302
|
+
bytes = Thread.handle_interrupt(Stopped => :never) { source.read(length, offset) }.to_s
|
|
303
|
+
return bytes if bytes.bytesize.eql?(length)
|
|
304
|
+
|
|
305
|
+
raise EOFError, format(CHANGED, source.description, source.size, index, bytes.bytesize, length, offset)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Upload a single chunk, sending it again after a server or network error
|
|
309
|
+
#
|
|
310
|
+
# A chunk names the segment it is appended at, so one sent twice is appended once, and it is sent again as
|
|
311
|
+
# {Utils.sending_again} sends a request again.
|
|
312
|
+
#
|
|
313
|
+
# @api private
|
|
314
|
+
# @param client [Client] the X API client
|
|
315
|
+
# @param media_id [String] the media ID
|
|
316
|
+
# @param upload_body [String] the upload body
|
|
317
|
+
# @param headers [Hash] the request headers
|
|
318
|
+
# @return [void]
|
|
319
|
+
def upload_chunk(client:, media_id:, upload_body:, headers:)
|
|
320
|
+
Utils.sending_again(client) { client.post("media/upload/#{media_id}/append", upload_body, headers:, **JSON_CLASSES) }
|
|
321
|
+
end
|
|
322
|
+
end
|
|
323
|
+
private_constant :Chunks
|
|
324
|
+
end
|
|
325
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "x/core"
|
|
4
|
+
|
|
5
|
+
module X
|
|
6
|
+
module Uploads
|
|
7
|
+
# Base error class for the failures of an upload, which every error x-uploads raises of its own descends from
|
|
8
|
+
#
|
|
9
|
+
# It descends from X::Error, so that rescuing the errors of the X API catches an upload failure as it did before.
|
|
10
|
+
# The errors that descend from it are named directly under X, as the errors of x-core are, so that this is the
|
|
11
|
+
# one name under X::Uploads a rescue reaches for.
|
|
12
|
+
#
|
|
13
|
+
# It catches the errors x-uploads raises of its own: media the API would refuse, raised before any request, and
|
|
14
|
+
# the failures after media was uploaded, which hold the media. It does not catch the X::Error of a request the
|
|
15
|
+
# API refused, or that got no response, before there was media to hold, such as the X::BadRequest of the request
|
|
16
|
+
# that initializes an upload, or of an image sent in a single request, nor an ArgumentError, which is not an
|
|
17
|
+
# X::Error: that of a mistake in the arguments of a call, and the one a request raises when the client was given
|
|
18
|
+
# no proxy_url and the proxy the environment names cannot be parsed, or is not an http or https URL with a host.
|
|
19
|
+
# Rescue X::Error to catch every failure of an
|
|
20
|
+
# upload but those.
|
|
21
|
+
#
|
|
22
|
+
# @api public
|
|
23
|
+
class Error < X::Error; end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "source"
|
|
4
|
+
|
|
5
|
+
module X
|
|
6
|
+
module Uploads
|
|
7
|
+
# Tells an animated GIF from a still one by reading its blocks, without decoding any image
|
|
8
|
+
#
|
|
9
|
+
# Internal to x-uploads: an upload tells with it whether a GIF is processed as a GIF or as an image, which X
|
|
10
|
+
# decides, so that it can change within 1.x as X does.
|
|
11
|
+
#
|
|
12
|
+
# @api private
|
|
13
|
+
module Gif
|
|
14
|
+
extend self
|
|
15
|
+
|
|
16
|
+
# Size in bytes of the header and logical screen descriptor
|
|
17
|
+
HEADER_SIZE = 13
|
|
18
|
+
# Byte that introduces an extension block
|
|
19
|
+
EXTENSION_INTRODUCER = 0x21
|
|
20
|
+
# Byte that introduces an image
|
|
21
|
+
IMAGE_SEPARATOR = 0x2C
|
|
22
|
+
# Size in bytes of an image descriptor, counting its separator
|
|
23
|
+
IMAGE_DESCRIPTOR_SIZE = 10
|
|
24
|
+
private_constant :HEADER_SIZE, :EXTENSION_INTRODUCER, :IMAGE_SEPARATOR, :IMAGE_DESCRIPTOR_SIZE
|
|
25
|
+
|
|
26
|
+
# Check whether a GIF holds more than one frame
|
|
27
|
+
#
|
|
28
|
+
# The GIF is read whole, so an upload asks it only of a GIF no larger than the API takes, which is 15 megabytes.
|
|
29
|
+
#
|
|
30
|
+
# @api private
|
|
31
|
+
# @param media [String, Pathname, IO, StringIO] the path to the GIF, or an IO open on it
|
|
32
|
+
# @return [Boolean] true if the GIF has a second frame
|
|
33
|
+
# @example Check whether a GIF is animated
|
|
34
|
+
# Uploads::Gif.animated?("cat.gif") # => true
|
|
35
|
+
def animated?(media)
|
|
36
|
+
data = Source.for(media).content
|
|
37
|
+
position = skip_color_table(HEADER_SIZE, data.getbyte(10).to_i)
|
|
38
|
+
frames = 0
|
|
39
|
+
while (block = data.getbyte(position))
|
|
40
|
+
frames += 1 if block.eql?(IMAGE_SEPARATOR)
|
|
41
|
+
return true if frames > 1
|
|
42
|
+
|
|
43
|
+
position = after_block(data, position, block)
|
|
44
|
+
end
|
|
45
|
+
false
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
# The position after a block, or the end of the data after the last block
|
|
51
|
+
# @api private
|
|
52
|
+
# @param data [String] the GIF data
|
|
53
|
+
# @param position [Integer] the position of the block
|
|
54
|
+
# @param block [Integer] the byte that introduces the block
|
|
55
|
+
# @return [Integer] the position after the block
|
|
56
|
+
def after_block(data, position, block)
|
|
57
|
+
case block
|
|
58
|
+
when EXTENSION_INTRODUCER then skip_sub_blocks(data, position + 2)
|
|
59
|
+
when IMAGE_SEPARATOR
|
|
60
|
+
skip_sub_blocks(data, skip_color_table(position + IMAGE_DESCRIPTOR_SIZE, data.getbyte(position + 9).to_i) + 1)
|
|
61
|
+
else data.bytesize
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# The position after a color table, if the flags say one follows
|
|
66
|
+
# @api private
|
|
67
|
+
# @param position [Integer] the position where a color table would start
|
|
68
|
+
# @param flags [Integer] the packed flags of the descriptor, zero when the data ends before them
|
|
69
|
+
# @return [Integer] the position after the color table
|
|
70
|
+
def skip_color_table(position, flags)
|
|
71
|
+
flags.anybits?(0x80) ? position + (3 << ((flags & 7) + 1)) : position
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The position after a run of data sub-blocks and its terminator
|
|
75
|
+
# @api private
|
|
76
|
+
# @param data [String] the GIF data
|
|
77
|
+
# @param position [Integer] the position of the first sub-block
|
|
78
|
+
# @return [Integer] the position after the terminator
|
|
79
|
+
def skip_sub_blocks(data, position)
|
|
80
|
+
while (size = data.getbyte(position).to_i).positive?
|
|
81
|
+
position += size + 1
|
|
82
|
+
end
|
|
83
|
+
position + 1
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
private_constant :Gif
|
|
87
|
+
end
|
|
88
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "error"
|
|
4
|
+
|
|
5
|
+
module X
|
|
6
|
+
# Raised for media the API would refuse, before any request: a file that does not exist, and media that cannot be
|
|
7
|
+
# read, holds nothing, or is larger than the API takes
|
|
8
|
+
#
|
|
9
|
+
# The media of an upload is often given by a user, so it is told apart from a mistake in the arguments of a call,
|
|
10
|
+
# such as a media category that does not exist, which raises ArgumentError: code that uploads what a user gives
|
|
11
|
+
# rescues this, and X::InvalidMediaType, which descends from it, for media of a type the API does not take, without
|
|
12
|
+
# rescuing the ArgumentError of its own mistakes. One whose cause is an error of the system, such as Errno::EMFILE
|
|
13
|
+
# or Errno::EIO, says the machine could not read the media, not that the media is wrong.
|
|
14
|
+
#
|
|
15
|
+
# It descends from X::Uploads::Error, and so from X::Error, so rescuing the failures of an upload catches it.
|
|
16
|
+
#
|
|
17
|
+
# @api public
|
|
18
|
+
class InvalidMedia < Uploads::Error; end
|
|
19
|
+
end
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "invalid_media"
|
|
4
|
+
|
|
5
|
+
module X
|
|
6
|
+
# Error raised when a file's MIME type cannot be determined or is unsupported
|
|
7
|
+
#
|
|
8
|
+
# It descends from X::InvalidMedia, so rescuing media the API would refuse catches it.
|
|
9
|
+
#
|
|
10
|
+
# @api public
|
|
11
|
+
class InvalidMediaType < InvalidMedia; end
|
|
12
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module X
|
|
4
|
+
module Uploads
|
|
5
|
+
# The classes the uploaders parse responses into, whatever parsing classes a client defaults to
|
|
6
|
+
#
|
|
7
|
+
# The uploaders read the responses they receive, and return them, as Hashes and Arrays, so a client whose
|
|
8
|
+
# default_object_class is another class, such as OpenStruct, still uploads.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
JSON_CLASSES = {array_class: Array, object_class: Hash}.freeze
|
|
12
|
+
private_constant :JSON_CLASSES
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "timeout"
|
|
4
|
+
require_relative "error"
|
|
5
|
+
require_relative "uploaded_media"
|
|
6
|
+
require_relative "media_processing_timeout"
|
|
7
|
+
|
|
8
|
+
module X
|
|
9
|
+
# Error raised when media is uploaded, but a check of its processing fails
|
|
10
|
+
#
|
|
11
|
+
# An upload awaits the processing of media X processes, such as a video, once the media is uploaded,
|
|
12
|
+
# so the media is not lost to a check that fails, as one the API answers with a server error, or one the
|
|
13
|
+
# on_response hook of the client raises: the error holds the media, which can be awaited again with
|
|
14
|
+
# await_processing. The error that failed the check is the cause, whose
|
|
15
|
+
# message the message ends with. A TokenReportFailed, which a check raises when save_tokens raised for the tokens of
|
|
16
|
+
# a refresh it made, is raised as it is, rather than as this error, so that the rescue of it that stores the tokens
|
|
17
|
+
# it holds catches it around an upload as around any other request.
|
|
18
|
+
#
|
|
19
|
+
# @api public
|
|
20
|
+
class MediaProcessingCheckFailed < Uploads::Error
|
|
21
|
+
# Await processing, raising this error, which holds the media, if a check fails
|
|
22
|
+
#
|
|
23
|
+
# Internal to x-uploads: an upload awaits the processing of the media it uploaded through it, and calls it with
|
|
24
|
+
# __send__, since it is private. A MediaProcessingTimeout, which holds the media itself, a Timeout::Error, as
|
|
25
|
+
# Timeout.timeout raises around the upload, a TokenReportFailed, which holds the tokens of a refresh that
|
|
26
|
+
# save_tokens raised for, and an exception that is not a StandardError, such as an Interrupt, are raised as they
|
|
27
|
+
# are.
|
|
28
|
+
#
|
|
29
|
+
# @api private
|
|
30
|
+
# @param media [UploadedMedia] the uploaded media
|
|
31
|
+
# @yield awaits the processing
|
|
32
|
+
# @return [Object] what the block returns
|
|
33
|
+
# @raise [MediaProcessingCheckFailed] if the block raises a StandardError other than a MediaProcessingTimeout, a
|
|
34
|
+
# Timeout::Error, or a TokenReportFailed, such as an error of the X API
|
|
35
|
+
# @raise [TokenReportFailed] if save_tokens raises for the tokens of a refresh the block made, with the tokens
|
|
36
|
+
# @raise [MediaProcessingTimeout] if the media is still processing once the processing timeout would pass
|
|
37
|
+
# @example Await the processing of an upload, keeping the media if a check fails
|
|
38
|
+
# X::MediaProcessingCheckFailed.__send__(:keeping, media) { Uploads::MediaUpload.await_processing(media, client:) }
|
|
39
|
+
def self.keeping(media)
|
|
40
|
+
yield
|
|
41
|
+
rescue MediaProcessingTimeout, Timeout::Error, TokenReportFailed
|
|
42
|
+
raise
|
|
43
|
+
rescue
|
|
44
|
+
raise new(media:)
|
|
45
|
+
end
|
|
46
|
+
private_class_method :keeping
|
|
47
|
+
|
|
48
|
+
# The media that was uploaded, as the upload response describes it
|
|
49
|
+
# @api public
|
|
50
|
+
# @return [UploadedMedia, nil] the uploaded media, or nil if none was given
|
|
51
|
+
# @example Await the processing of the media again later
|
|
52
|
+
# rescue X::MediaProcessingCheckFailed => e
|
|
53
|
+
# client.await_media_processing(e.media)
|
|
54
|
+
attr_reader :media
|
|
55
|
+
|
|
56
|
+
# Initialize the error with the media that was uploaded
|
|
57
|
+
#
|
|
58
|
+
# The message is the one given, or else names the media by its identifier, when media that holds one was given.
|
|
59
|
+
#
|
|
60
|
+
# @api public
|
|
61
|
+
# @param message [String, nil] the message, or nil for one that names the media
|
|
62
|
+
# @param media [UploadedMedia, Hash{String => Object}, nil] the media that was uploaded, a Hash of which is held as
|
|
63
|
+
# uploaded media
|
|
64
|
+
# @return [MediaProcessingCheckFailed] a new error
|
|
65
|
+
# @example Raise the error for media whose processing could not be checked
|
|
66
|
+
# raise X::MediaProcessingCheckFailed.new(media: media)
|
|
67
|
+
# @example Raise the error with a message of its own, as a test stub may
|
|
68
|
+
# raise X::MediaProcessingCheckFailed, "Processing could not be checked"
|
|
69
|
+
def initialize(message = nil, media: nil)
|
|
70
|
+
@media = media.is_a?(Hash) ? UploadedMedia.new(media) : media
|
|
71
|
+
super(message || ["Media", media&.[]("id"), "was uploaded, but its processing could not be checked"].compact.join(" "))
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The message, ending with why the processing could not be checked
|
|
75
|
+
#
|
|
76
|
+
# It ends with the message of the error that failed the check, which is the cause, if there is one, whether the
|
|
77
|
+
# message was given or named the media.
|
|
78
|
+
#
|
|
79
|
+
# @api public
|
|
80
|
+
# @return [String] the message
|
|
81
|
+
# @example Read why the processing could not be checked
|
|
82
|
+
# error.message # => "Media 7 was uploaded, but its processing could not be checked: Service Unavailable"
|
|
83
|
+
def to_s = [super, cause&.message].compact.join(": ")
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "error"
|
|
4
|
+
require_relative "uploaded_media"
|
|
5
|
+
|
|
6
|
+
module X
|
|
7
|
+
# Error raised when X fails to process uploaded media, such as a video it cannot decode
|
|
8
|
+
# @api public
|
|
9
|
+
class MediaProcessingFailed < Uploads::Error
|
|
10
|
+
# The message of a failure X gives no reason for
|
|
11
|
+
DEFAULT_MESSAGE = "Media processing failed"
|
|
12
|
+
private_constant :DEFAULT_MESSAGE
|
|
13
|
+
|
|
14
|
+
# The uploaded media, as the processing status X reported describes it
|
|
15
|
+
#
|
|
16
|
+
# Its processing_info holds the state X reported, and the error, when X reported one.
|
|
17
|
+
#
|
|
18
|
+
# @api public
|
|
19
|
+
# @return [UploadedMedia, nil] the media, which reads as a Hash, or nil if none was given
|
|
20
|
+
# @example Read the state X reported
|
|
21
|
+
# error.media.state # => "failed"
|
|
22
|
+
attr_reader :media
|
|
23
|
+
|
|
24
|
+
# Initialize the error with the reason X gives for the failure
|
|
25
|
+
#
|
|
26
|
+
# The message is the one given, or else the reason the processing status holds, or else "Media processing
|
|
27
|
+
# failed".
|
|
28
|
+
#
|
|
29
|
+
# @api public
|
|
30
|
+
# @param message [String, nil] the message, or nil for the reason the processing status holds
|
|
31
|
+
# @param media [UploadedMedia, Hash{String => Object}, nil] the media, as the processing status X reported
|
|
32
|
+
# describes it, a Hash of which is held as uploaded media
|
|
33
|
+
# @return [MediaProcessingFailed] a new error
|
|
34
|
+
# @example Raise the error for a failed status
|
|
35
|
+
# raise X::MediaProcessingFailed.new(media: status)
|
|
36
|
+
# @example Raise the error with a message of its own, as a test stub may
|
|
37
|
+
# raise X::MediaProcessingFailed, "Unsupported video format"
|
|
38
|
+
def initialize(message = nil, media: nil)
|
|
39
|
+
@media = media.is_a?(Hash) ? UploadedMedia.new(media) : media
|
|
40
|
+
super(message || media&.dig("processing_info", "error", "message") || DEFAULT_MESSAGE)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|