kompressr-sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/API.md +120 -0
- data/CHANGELOG.md +10 -0
- data/LICENSE +21 -0
- data/PARITY.md +32 -0
- data/README.md +149 -0
- data/examples/private_upload.rb +42 -0
- data/examples/rails_service.rb +34 -0
- data/lib/kompressr/client.rb +197 -0
- data/lib/kompressr/error.rb +49 -0
- data/lib/kompressr/platform_client.rb +112 -0
- data/lib/kompressr/transport.rb +528 -0
- data/lib/kompressr/validation.rb +278 -0
- data/lib/kompressr/version.rb +5 -0
- data/lib/kompressr.rb +8 -0
- metadata +179 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kompressr
|
|
4
|
+
# Safe recovery metadata. Raw response bodies and request credentials are absent.
|
|
5
|
+
class Error < StandardError
|
|
6
|
+
attr_reader :kind, :code, :status, :retry_after_ms, :asset_id, :may_have_succeeded
|
|
7
|
+
|
|
8
|
+
def initialize(message, kind:, code:, status: nil, retry_after_ms: nil, asset_id: nil, may_have_succeeded: false)
|
|
9
|
+
super(message)
|
|
10
|
+
@kind, @code, @status = kind, code, status
|
|
11
|
+
@retry_after_ms, @asset_id, @may_have_succeeded = retry_after_ms, asset_id, may_have_succeeded
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def to_h
|
|
15
|
+
{ 'message' => message, 'kind' => kind, 'code' => code, 'status' => status,
|
|
16
|
+
'retry_after_ms' => retry_after_ms, 'asset_id' => asset_id, 'may_have_succeeded' => may_have_succeeded }
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def to_json(*args) = to_h.to_json(*args)
|
|
20
|
+
def inspect = "#<Kompressr::Error #{to_h.inspect}>"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Thread-safe cooperative cancellation. Cancellation never rolls back server work.
|
|
24
|
+
class CancellationToken
|
|
25
|
+
def initialize
|
|
26
|
+
@mutex = Mutex.new
|
|
27
|
+
@cancelled = false
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def cancel = @mutex.synchronize { @cancelled = true }
|
|
31
|
+
def cancelled? = @mutex.synchronize { @cancelled }
|
|
32
|
+
def inspect = "#<Kompressr::CancellationToken cancelled=#{cancelled?}>"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Provisioning secrets require explicit access and are excluded from normal logs.
|
|
36
|
+
class IssuedKey
|
|
37
|
+
attr_reader :id, :key_id, :upload_key
|
|
38
|
+
|
|
39
|
+
def initialize(id:, key_id:, api_key:, upload_key: nil)
|
|
40
|
+
@id, @key_id, @api_key, @upload_key = id, key_id, api_key, upload_key
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def reveal_api_key = @api_key.dup
|
|
44
|
+
def to_h = { 'id' => id, 'key_id' => key_id, 'upload_key' => upload_key, 'api_key' => '[redacted]' }
|
|
45
|
+
def to_json(*args) = to_h.to_json(*args)
|
|
46
|
+
def inspect = "#<Kompressr::IssuedKey #{to_h.inspect}>"
|
|
47
|
+
alias to_s inspect
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Kompressr
|
|
4
|
+
# Project authority is checked by the service, never inferred by this client.
|
|
5
|
+
class PlatformClient
|
|
6
|
+
def initialize(**configuration)
|
|
7
|
+
@transport = Internal::Transport.new(**configuration)
|
|
8
|
+
end
|
|
9
|
+
def inspect = '#<Kompressr::PlatformClient credentials redacted>'
|
|
10
|
+
alias to_s inspect
|
|
11
|
+
|
|
12
|
+
def list_users(page: 1, limit: 20, timeout: nil, cancel: nil)
|
|
13
|
+
query = pagination(page, limit)
|
|
14
|
+
@transport.request("/api/platform/project-users?#{URI.encode_www_form(query)}", timeout: timeout, cancel: cancel) do |value|
|
|
15
|
+
response = Validation.success(value)
|
|
16
|
+
result = Validation.page(response, 'project_users') { |row| Validation.project_user(row) }
|
|
17
|
+
%w[project_active_count org_active_count].each { |key| result[key] = Validation.nonnegative(response[key]) }
|
|
18
|
+
if response.key?('usage_period')
|
|
19
|
+
period = Validation.record(response['usage_period'])
|
|
20
|
+
%w[start end].each { |key| Validation.bad unless /\A\d{4}-\d{2}-\d{2}\z/.match?(Validation.string(period[key])) }
|
|
21
|
+
result['usage_period'] = Validation.select(period, %w[start end])
|
|
22
|
+
end
|
|
23
|
+
result
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def create_user(customer_user_id, display_name: nil, metadata: nil, timeout: nil, cancel: nil)
|
|
28
|
+
customer_user_id = Validation.text(customer_user_id, 'customer_user_id', 256)
|
|
29
|
+
body = user_fields(display_name, metadata).merge('customer_user_id' => customer_user_id)
|
|
30
|
+
@transport.request('/api/platform/project-users', method: 'POST', body: body, timeout: timeout, cancel: cancel) do |value|
|
|
31
|
+
response = Validation.success(value)
|
|
32
|
+
{ 'created' => Validation.boolean(response['created']), 'project_user' => Validation.project_user(response['project_user']) }
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def update_user(project_user_id, display_name: nil, metadata: nil, timeout: nil, cancel: nil)
|
|
37
|
+
body = user_fields(display_name, metadata)
|
|
38
|
+
@transport.request(user_path(project_user_id), method: 'PATCH', body: body, timeout: timeout, cancel: cancel) do |value|
|
|
39
|
+
Validation.project_user(Validation.success(value)['project_user'])
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def suspend_user(project_user_id, timeout: nil, cancel: nil)
|
|
44
|
+
mutate_user(project_user_id, 'suspend', timeout, cancel)
|
|
45
|
+
end
|
|
46
|
+
def reactivate_user(project_user_id, timeout: nil, cancel: nil)
|
|
47
|
+
mutate_user(project_user_id, 'reactivate', timeout, cancel)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def list_upload_keys(project_user_id, page: 1, limit: 20, timeout: nil, cancel: nil)
|
|
51
|
+
query = URI.encode_www_form(pagination(page, limit))
|
|
52
|
+
@transport.request("#{user_path(project_user_id)}/upload-keys?#{query}", timeout: timeout, cancel: cancel) do |value|
|
|
53
|
+
Validation.page(Validation.success(value), 'upload_keys') { |row| Validation.upload_key(row) }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def create_upload_key(project_user_id, name, scopes: nil, allowed_visibility: nil, allowed_asset_types: nil,
|
|
58
|
+
expires_at: nil, timeout: nil, cancel: nil)
|
|
59
|
+
path = user_path(project_user_id)
|
|
60
|
+
name = Validation.text(name, 'name', 120)
|
|
61
|
+
body = { 'name' => name }
|
|
62
|
+
unless scopes.nil?
|
|
63
|
+
Validation.invalid('Supply at least one scope, or omit scopes to use server defaults.') unless scopes.instance_of?(Array) && !scopes.empty?
|
|
64
|
+
scopes.each { |scope| Validation.enum(scope, Validation::SCOPES, 'upload scope') }
|
|
65
|
+
body['scopes'] = scopes
|
|
66
|
+
end
|
|
67
|
+
body['allowed_visibility'] = Validation.enum(allowed_visibility, %w[public private both], 'allowed_visibility') unless allowed_visibility.nil?
|
|
68
|
+
unless allowed_asset_types.nil?
|
|
69
|
+
Validation.invalid('allowed_asset_types must be an array of positive integer IDs.') unless allowed_asset_types.instance_of?(Array)
|
|
70
|
+
allowed_asset_types.each { |type| Validation.positive(type, 'asset type') }
|
|
71
|
+
body['allowed_asset_types'] = allowed_asset_types
|
|
72
|
+
end
|
|
73
|
+
unless expires_at.nil?
|
|
74
|
+
begin
|
|
75
|
+
Validation.invalid('expires_at must be a future ISO timestamp with a timezone.') unless expires_at.instance_of?(String) && expires_at.ascii_only? && /(?:Z|[+-]\d{2}:\d{2})\z/.match?(expires_at) && Time.iso8601(expires_at) > Time.now
|
|
76
|
+
rescue ArgumentError
|
|
77
|
+
Validation.invalid('expires_at must be a future ISO timestamp with a timezone.')
|
|
78
|
+
end
|
|
79
|
+
body['expires_at'] = expires_at
|
|
80
|
+
end
|
|
81
|
+
@transport.request("#{path}/upload-keys", method: 'POST', body: body, timeout: timeout, cancel: cancel) { |v| Validation.issued(v) }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def rotate_upload_key(project_user_id, key_id, timeout: nil, cancel: nil)
|
|
85
|
+
path = "#{user_path(project_user_id)}/upload-keys/#{Validation.uuid(key_id, 'key_id')}/rotate"
|
|
86
|
+
@transport.request(path, method: 'POST', timeout: timeout, cancel: cancel) { |v| Validation.issued(v) }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def revoke_upload_key(project_user_id, key_id, timeout: nil, cancel: nil)
|
|
90
|
+
path = "#{user_path(project_user_id)}/upload-keys/#{Validation.uuid(key_id, 'key_id')}/revoke"
|
|
91
|
+
@transport.request(path, method: 'POST', timeout: timeout, cancel: cancel) { |v| Validation.success(v); nil }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
private
|
|
95
|
+
|
|
96
|
+
def pagination(page, limit)
|
|
97
|
+
{ 'page' => Validation.positive(page, 'page', 2_147_483_647), 'limit' => Validation.positive(limit, 'limit', 100) }
|
|
98
|
+
end
|
|
99
|
+
def user_path(id) = "/api/platform/project-users/#{Validation.uuid(id, 'project_user_id')}"
|
|
100
|
+
def user_fields(display_name, metadata)
|
|
101
|
+
body = {}
|
|
102
|
+
body['display_name'] = Validation.text(display_name, 'display_name', 120, empty: true) unless display_name.nil?
|
|
103
|
+
body['metadata'] = Validation.customer_data(metadata, 8192) unless metadata.nil?
|
|
104
|
+
body
|
|
105
|
+
end
|
|
106
|
+
def mutate_user(id, action, timeout, cancel)
|
|
107
|
+
@transport.request("#{user_path(id)}/#{action}", method: 'POST', timeout: timeout, cancel: cancel) do |value|
|
|
108
|
+
Validation.project_user(Validation.success(value)['project_user'])
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,528 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'socket'
|
|
4
|
+
require 'openssl'
|
|
5
|
+
require 'ipaddr'
|
|
6
|
+
require 'securerandom'
|
|
7
|
+
|
|
8
|
+
module Kompressr
|
|
9
|
+
module Internal
|
|
10
|
+
# Only DNS runs in background threads. A worker never owns a connection or a
|
|
11
|
+
# credential, and cannot dispatch a request after its caller has left.
|
|
12
|
+
module Capacity
|
|
13
|
+
@mutex = Mutex.new
|
|
14
|
+
@pid = Process.pid
|
|
15
|
+
@counts = { dns: 0, connection: 0 }
|
|
16
|
+
LIMITS = { dns: 8, connection: 64 }.freeze
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
def acquire(kind, context)
|
|
20
|
+
loop do
|
|
21
|
+
context.check!
|
|
22
|
+
taken = @mutex.synchronize do
|
|
23
|
+
if @pid != Process.pid
|
|
24
|
+
@pid = Process.pid
|
|
25
|
+
@counts = { dns: 0, connection: 0 }
|
|
26
|
+
end
|
|
27
|
+
if @counts[kind] < LIMITS[kind]
|
|
28
|
+
@counts[kind] += 1
|
|
29
|
+
true
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
return if taken
|
|
33
|
+
context.pause(0.01)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def release(kind) = @mutex.synchronize { @counts[kind] -= 1 }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
class Context
|
|
41
|
+
attr_accessor :status, :retry_after_ms, :asset_id, :dispatched
|
|
42
|
+
attr_reader :deadline
|
|
43
|
+
|
|
44
|
+
def initialize(timeout, cancel, write, asset_id)
|
|
45
|
+
@deadline = self.class.now + timeout
|
|
46
|
+
@cancel, @write, @asset_id = cancel, write, asset_id
|
|
47
|
+
@dispatched = false
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def self.now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
51
|
+
def remaining = @deadline - self.class.now
|
|
52
|
+
def error(kind)
|
|
53
|
+
code, message = {
|
|
54
|
+
'timeout' => ['REQUEST_TIMEOUT', 'Request timed out; inspect a write outcome before retrying.'],
|
|
55
|
+
'cancelled' => ['REQUEST_CANCELLED', 'Request cancelled; accepted server work is not rolled back.'],
|
|
56
|
+
'network' => ['NETWORK_ERROR', 'Unable to reach Kompressr; check the endpoint and connection.']
|
|
57
|
+
}.fetch(kind)
|
|
58
|
+
Error.new(message, kind: kind, code: code, status: @status, retry_after_ms: @retry_after_ms,
|
|
59
|
+
asset_id: @asset_id, may_have_succeeded: @write && @dispatched)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def check!
|
|
63
|
+
raise error('cancelled'), cause: nil if @cancel&.cancelled?
|
|
64
|
+
raise error('timeout'), cause: nil if remaining <= 0
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def pause(seconds)
|
|
68
|
+
# Compare remaining time before adding a potentially enormous retry delay.
|
|
69
|
+
stop = self.class.now + [seconds, [remaining, 0].max].min
|
|
70
|
+
loop do
|
|
71
|
+
check!
|
|
72
|
+
left = stop - self.class.now
|
|
73
|
+
break if left <= 0
|
|
74
|
+
sleep([left, 0.01].min)
|
|
75
|
+
end
|
|
76
|
+
check!
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def wait(io, direction, phase_deadline = @deadline, also_read: false)
|
|
80
|
+
check!
|
|
81
|
+
left = [remaining, phase_deadline - self.class.now].min
|
|
82
|
+
raise PhaseTimeout if left <= 0
|
|
83
|
+
delay = [left, 0.01].min
|
|
84
|
+
if direction == :wait_readable
|
|
85
|
+
IO.select([io], nil, nil, delay)
|
|
86
|
+
else
|
|
87
|
+
IO.select(also_read ? [io] : nil, [io], nil, delay)
|
|
88
|
+
end
|
|
89
|
+
check!
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
class PhaseTimeout < StandardError; end
|
|
94
|
+
|
|
95
|
+
module Network
|
|
96
|
+
module_function
|
|
97
|
+
|
|
98
|
+
def addresses(hostname, port, context)
|
|
99
|
+
begin
|
|
100
|
+
IPAddr.new(hostname)
|
|
101
|
+
return Addrinfo.getaddrinfo(hostname, port, nil, Socket::SOCK_STREAM, nil, Socket::AI_NUMERICHOST)
|
|
102
|
+
rescue IPAddr::InvalidAddressError
|
|
103
|
+
# Domain lookup is isolated from the request thread and its sockets.
|
|
104
|
+
end
|
|
105
|
+
Capacity.acquire(:dns, context)
|
|
106
|
+
result = Queue.new
|
|
107
|
+
begin
|
|
108
|
+
Thread.new do
|
|
109
|
+
Thread.current.report_on_exception = false
|
|
110
|
+
begin
|
|
111
|
+
result << [:ok, Addrinfo.getaddrinfo(hostname, port, nil, Socket::SOCK_STREAM)]
|
|
112
|
+
rescue StandardError
|
|
113
|
+
result << [:error, nil]
|
|
114
|
+
ensure
|
|
115
|
+
Capacity.release(:dns)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
rescue StandardError
|
|
119
|
+
Capacity.release(:dns)
|
|
120
|
+
raise
|
|
121
|
+
end
|
|
122
|
+
loop do
|
|
123
|
+
context.check!
|
|
124
|
+
begin
|
|
125
|
+
state, value = result.pop(true)
|
|
126
|
+
raise SocketError, 'DNS lookup failed.' unless state == :ok
|
|
127
|
+
return value
|
|
128
|
+
rescue ThreadError
|
|
129
|
+
context.pause(0.01)
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def connect(uri, context)
|
|
135
|
+
addresses = addresses(uri.hostname, uri.port, context).uniq(&:to_sockaddr)
|
|
136
|
+
raise SocketError, 'No addresses returned.' if addresses.empty?
|
|
137
|
+
addresses.each_with_index do |address, index|
|
|
138
|
+
context.check!
|
|
139
|
+
socket = Socket.new(address.afamily, Socket::SOCK_STREAM, 0)
|
|
140
|
+
keep = false
|
|
141
|
+
begin
|
|
142
|
+
phase_deadline = Context.now + context.remaining / (addresses.size - index)
|
|
143
|
+
loop do
|
|
144
|
+
context.check!
|
|
145
|
+
result = socket.connect_nonblock(address.to_sockaddr, exception: false)
|
|
146
|
+
break if result == 0
|
|
147
|
+
context.wait(socket, result, phase_deadline)
|
|
148
|
+
rescue Errno::EISCONN
|
|
149
|
+
break
|
|
150
|
+
end
|
|
151
|
+
context.check!
|
|
152
|
+
if uri.scheme == 'https'
|
|
153
|
+
ssl_context = OpenSSL::SSL::SSLContext.new
|
|
154
|
+
ssl_context.set_params(verify_mode: OpenSSL::SSL::VERIFY_PEER,
|
|
155
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION)
|
|
156
|
+
ssl = OpenSSL::SSL::SSLSocket.new(socket, ssl_context)
|
|
157
|
+
ssl.sync_close = true
|
|
158
|
+
ssl.hostname = uri.hostname
|
|
159
|
+
socket = ssl
|
|
160
|
+
loop do
|
|
161
|
+
context.check!
|
|
162
|
+
result = socket.connect_nonblock(exception: false)
|
|
163
|
+
break unless result == :wait_readable || result == :wait_writable
|
|
164
|
+
context.wait(socket, result, phase_deadline)
|
|
165
|
+
end
|
|
166
|
+
socket.post_connection_check(uri.hostname)
|
|
167
|
+
end
|
|
168
|
+
context.check!
|
|
169
|
+
keep = true
|
|
170
|
+
return socket
|
|
171
|
+
rescue PhaseTimeout, SystemCallError, IOError
|
|
172
|
+
raise if index == addresses.size - 1
|
|
173
|
+
ensure
|
|
174
|
+
socket.close unless keep || socket.closed?
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
raise SocketError, 'Connection failed.'
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
class Body
|
|
182
|
+
attr_reader :content_type, :length
|
|
183
|
+
def initialize(content)
|
|
184
|
+
@content = content.b.freeze
|
|
185
|
+
@length = @content.bytesize
|
|
186
|
+
@content_type = 'application/json'
|
|
187
|
+
end
|
|
188
|
+
def each
|
|
189
|
+
offset = 0
|
|
190
|
+
while offset < @length
|
|
191
|
+
yield @content.byteslice(offset, 65_536)
|
|
192
|
+
offset += 65_536
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
class Multipart
|
|
198
|
+
attr_reader :content_type, :length
|
|
199
|
+
def initialize(source, size, filename, fields)
|
|
200
|
+
@source, @size = source, size
|
|
201
|
+
boundary = "kompressr-#{SecureRandom.hex(24)}"
|
|
202
|
+
@content_type = "multipart/form-data; boundary=#{boundary}"
|
|
203
|
+
@prefix = fields.map { |name, value| "--#{boundary}\r\nContent-Disposition: form-data; name=\"#{name}\"\r\n\r\n#{value}\r\n" }.join.b
|
|
204
|
+
escaped = filename.gsub('"', '%22')
|
|
205
|
+
@prefix << "--#{boundary}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"#{escaped}\"\r\nContent-Type: application/octet-stream\r\n\r\n".b
|
|
206
|
+
@suffix = "\r\n--#{boundary}--\r\n".b
|
|
207
|
+
@length = @prefix.bytesize + @size + @suffix.bytesize
|
|
208
|
+
end
|
|
209
|
+
def each
|
|
210
|
+
yield @prefix
|
|
211
|
+
offset = 0
|
|
212
|
+
while offset < @size
|
|
213
|
+
chunk = @source.instance_of?(String) ? @source.byteslice(offset, [65_536, @size - offset].min) : @source.read([65_536, @size - offset].min)
|
|
214
|
+
raise IOError, 'Upload source changed while reading.' if chunk.nil? || chunk.empty?
|
|
215
|
+
yield chunk
|
|
216
|
+
offset += chunk.bytesize
|
|
217
|
+
end
|
|
218
|
+
yield @suffix
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# One fresh HTTP/1.1 connection per attempt. Identity bodies only, with strict
|
|
223
|
+
# framing and bounded headers/body; no redirect or native-library replay path.
|
|
224
|
+
class Wire
|
|
225
|
+
MAX_BODY = 2 * 1024 * 1024
|
|
226
|
+
MAX_HEADER = 65_536
|
|
227
|
+
TOKEN = /\A[!#$%&'*+.^_`|~0-9A-Za-z-]+\z/
|
|
228
|
+
|
|
229
|
+
def initialize(socket, context)
|
|
230
|
+
@socket, @context = socket, context
|
|
231
|
+
@buffer = ''.b
|
|
232
|
+
@informational = 0
|
|
233
|
+
@eof = false
|
|
234
|
+
end
|
|
235
|
+
def write(bytes, watch_response: false)
|
|
236
|
+
offset = 0
|
|
237
|
+
while offset < bytes.bytesize
|
|
238
|
+
@context.check!
|
|
239
|
+
return :early_response if watch_response && poll_response
|
|
240
|
+
@context.dispatched = true
|
|
241
|
+
result = @socket.write_nonblock(bytes.byteslice(offset, bytes.bytesize - offset), exception: false)
|
|
242
|
+
if result == :wait_readable || result == :wait_writable
|
|
243
|
+
@context.wait(@socket, result, also_read: watch_response)
|
|
244
|
+
else
|
|
245
|
+
raise IOError, 'Socket write stopped.' unless result.positive?
|
|
246
|
+
offset += result
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
nil
|
|
250
|
+
rescue SystemCallError, IOError, OpenSSL::SSL::SSLError
|
|
251
|
+
# A server can send a complete denial and close while the client writes.
|
|
252
|
+
# Read its pending response before classifying a body-write failure.
|
|
253
|
+
return :early_response if watch_response
|
|
254
|
+
raise
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def poll_response
|
|
258
|
+
loop do
|
|
259
|
+
@context.check!
|
|
260
|
+
if (line_end = @buffer.index("\r\n"))
|
|
261
|
+
status = parse_status(@buffer.byteslice(0, line_end))
|
|
262
|
+
# Once a final status line is available, stop sending the body and
|
|
263
|
+
# finish reading its bounded headers/body, even if they arrive slowly.
|
|
264
|
+
return true if status >= 200
|
|
265
|
+
if @buffer.include?("\r\n\r\n")
|
|
266
|
+
read_head
|
|
267
|
+
next
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
Validation.bad('API response headers exceed the supported size.') if @buffer.bytesize > MAX_HEADER + 8192
|
|
271
|
+
result = @socket.read_nonblock(16_384, exception: false)
|
|
272
|
+
return false if result == :wait_readable || result == :wait_writable
|
|
273
|
+
if result.nil?
|
|
274
|
+
@eof = true
|
|
275
|
+
return true
|
|
276
|
+
end
|
|
277
|
+
@buffer << result
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
def read_chunk
|
|
281
|
+
return nil if @eof
|
|
282
|
+
loop do
|
|
283
|
+
@context.check!
|
|
284
|
+
result = @socket.read_nonblock(16_384, exception: false)
|
|
285
|
+
if result == :wait_readable || result == :wait_writable
|
|
286
|
+
@context.wait(@socket, result)
|
|
287
|
+
else
|
|
288
|
+
return result
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
end
|
|
292
|
+
def line(maximum = MAX_HEADER)
|
|
293
|
+
loop do
|
|
294
|
+
@context.check!
|
|
295
|
+
if (index = @buffer.index("\r\n"))
|
|
296
|
+
Validation.bad('API response headers exceed the supported size.') if index + 2 > maximum
|
|
297
|
+
result = @buffer.slice!(0, index + 2)
|
|
298
|
+
return result.byteslice(0, result.bytesize - 2)
|
|
299
|
+
end
|
|
300
|
+
Validation.bad('API response headers exceed the supported size.') if @buffer.bytesize >= maximum
|
|
301
|
+
chunk = read_chunk
|
|
302
|
+
raise EOFError, 'Truncated HTTP response.' if chunk.nil?
|
|
303
|
+
@buffer << chunk
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
def exact(length)
|
|
307
|
+
while @buffer.bytesize < length
|
|
308
|
+
chunk = read_chunk
|
|
309
|
+
raise EOFError, 'Truncated HTTP response.' if chunk.nil?
|
|
310
|
+
@buffer << chunk
|
|
311
|
+
end
|
|
312
|
+
@buffer.slice!(0, length)
|
|
313
|
+
end
|
|
314
|
+
def headers(track_response: false)
|
|
315
|
+
result = Hash.new { |h, k| h[k] = [] }
|
|
316
|
+
consumed = 0
|
|
317
|
+
loop do
|
|
318
|
+
row = line(MAX_HEADER - consumed)
|
|
319
|
+
consumed += row.bytesize + 2
|
|
320
|
+
break if row.empty?
|
|
321
|
+
name, value = row.split(':', 2)
|
|
322
|
+
Validation.bad('Malformed HTTP response headers.') unless name && TOKEN.match?(name) && value && !/[\x00-\x08\x0a-\x1f\x7f]/.match?(value)
|
|
323
|
+
result[name.downcase] << value.strip
|
|
324
|
+
@context.retry_after_ms = Transport.retry_after(result, nil) if track_response && name.downcase == 'retry-after'
|
|
325
|
+
end
|
|
326
|
+
result
|
|
327
|
+
end
|
|
328
|
+
def parse_status(status_line)
|
|
329
|
+
Validation.bad('API response status exceeds the supported size.') if status_line.bytesize + 2 > 8192
|
|
330
|
+
match = /\AHTTP\/1\.[01] ([1-5][0-9]{2})(?: [\x20-\x7e\t]*)?\z/.match(status_line)
|
|
331
|
+
Validation.bad('Malformed HTTP response status.') unless match
|
|
332
|
+
match[1].to_i
|
|
333
|
+
end
|
|
334
|
+
def read_head
|
|
335
|
+
status = parse_status(line(8192))
|
|
336
|
+
@context.status = status if status >= 200
|
|
337
|
+
head = headers(track_response: status >= 200)
|
|
338
|
+
if status < 200
|
|
339
|
+
@informational += 1
|
|
340
|
+
Validation.bad('Unsupported HTTP protocol upgrade or informational response.') if status == 101 || @informational > 5 || head.key?('content-length') || head.key?('transfer-encoding')
|
|
341
|
+
end
|
|
342
|
+
[status, head]
|
|
343
|
+
end
|
|
344
|
+
def response
|
|
345
|
+
loop do
|
|
346
|
+
status, head = read_head
|
|
347
|
+
if status < 200
|
|
348
|
+
next
|
|
349
|
+
end
|
|
350
|
+
@context.retry_after_ms = Transport.retry_after(head, nil)
|
|
351
|
+
return [status, head, body(status, head)]
|
|
352
|
+
end
|
|
353
|
+
end
|
|
354
|
+
def body(status, head)
|
|
355
|
+
encodings = head.fetch('content-encoding', []).flat_map { |value| value.split(',', -1) }
|
|
356
|
+
Validation.bad('API response uses an unsupported content encoding.') if encodings.any? { |value| value.strip.downcase != 'identity' }
|
|
357
|
+
lengths = head.fetch('content-length', [])
|
|
358
|
+
transfers = head.fetch('transfer-encoding', []).flat_map { |value| value.split(',', -1).map { |v| v.strip.downcase } }
|
|
359
|
+
Validation.bad('Ambiguous HTTP response framing.') if lengths.size > 1 || (!lengths.empty? && !transfers.empty?)
|
|
360
|
+
Validation.bad('Unsupported HTTP transfer encoding.') unless transfers.empty? || transfers == ['chunked']
|
|
361
|
+
length = nil
|
|
362
|
+
unless lengths.empty?
|
|
363
|
+
Validation.bad('Invalid response Content-Length.') unless /\A[0-9]+\z/.match?(lengths.first)
|
|
364
|
+
length = lengths.first.to_i
|
|
365
|
+
Validation.bad('API response exceeds the supported size.') if length > MAX_BODY
|
|
366
|
+
end
|
|
367
|
+
return ''.b if status == 204 || status == 304
|
|
368
|
+
return exact(length) if length
|
|
369
|
+
output = ''.b
|
|
370
|
+
if transfers == ['chunked']
|
|
371
|
+
loop do
|
|
372
|
+
chunk_line = line(8192)
|
|
373
|
+
match = /\A([0-9a-fA-F]+)(?:;[\x20-\x7e\t]*)?\z/.match(chunk_line)
|
|
374
|
+
Validation.bad('Invalid chunked response.') unless match
|
|
375
|
+
size = match[1].to_i(16)
|
|
376
|
+
if size.zero?
|
|
377
|
+
trailers = headers
|
|
378
|
+
Validation.bad('Invalid response trailers.') if %w[content-length transfer-encoding content-encoding host].any? { |name| trailers.key?(name) }
|
|
379
|
+
break
|
|
380
|
+
end
|
|
381
|
+
Validation.bad('API response exceeds the supported size.') if size > MAX_BODY - output.bytesize
|
|
382
|
+
output << exact(size)
|
|
383
|
+
Validation.bad('Invalid chunked response.') unless exact(2) == "\r\n"
|
|
384
|
+
end
|
|
385
|
+
else
|
|
386
|
+
output << @buffer
|
|
387
|
+
@buffer = ''.b
|
|
388
|
+
loop do
|
|
389
|
+
Validation.bad('API response exceeds the supported size.') if output.bytesize > MAX_BODY
|
|
390
|
+
chunk = read_chunk
|
|
391
|
+
break if chunk.nil?
|
|
392
|
+
Validation.bad('API response exceeds the supported size.') if chunk.bytesize > MAX_BODY - output.bytesize
|
|
393
|
+
output << chunk
|
|
394
|
+
end
|
|
395
|
+
end
|
|
396
|
+
output
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
|
|
400
|
+
class Transport
|
|
401
|
+
attr_reader :api_base_url, :upload_base_url, :request_timeout
|
|
402
|
+
|
|
403
|
+
def initialize(api_key:, api_base_url: 'https://kompressr.com', upload_base_url: nil,
|
|
404
|
+
request_timeout: 30, upload_timeout: 600, max_read_retries: 2)
|
|
405
|
+
Validation.invalid('Supply a nonempty ASCII API key on your trusted backend.') unless api_key.instance_of?(String) && api_key.ascii_only? && !api_key.strip.empty? && /\A[\x20-\x7e]+\z/.match?(api_key)
|
|
406
|
+
@key = api_key.dup.freeze
|
|
407
|
+
@api_base_url = Validation.origin(api_base_url)
|
|
408
|
+
@upload_base_url = Validation.origin(upload_base_url.nil? ? (@api_base_url == 'https://kompressr.com' ? 'https://uploads.kompressr.com' : @api_base_url) : upload_base_url)
|
|
409
|
+
@request_timeout = Validation.seconds(request_timeout, 'request_timeout')
|
|
410
|
+
@upload_timeout = Validation.seconds(upload_timeout, 'upload_timeout')
|
|
411
|
+
Validation.invalid('max_read_retries must be an integer from zero to five.') unless max_read_retries.instance_of?(Integer) && max_read_retries.between?(0, 5)
|
|
412
|
+
@retries = max_read_retries
|
|
413
|
+
end
|
|
414
|
+
def inspect = '#<Kompressr::Internal::Transport credentials redacted>'
|
|
415
|
+
alias to_s inspect
|
|
416
|
+
|
|
417
|
+
def self.retry_after(headers, parsed)
|
|
418
|
+
value = headers.fetch('retry-after', []).first
|
|
419
|
+
if value
|
|
420
|
+
if /\A[0-9]+(?:\.[0-9]+)?\z/.match?(value)
|
|
421
|
+
return [value.to_f, Float::MAX / 1000].min * 1000
|
|
422
|
+
end
|
|
423
|
+
begin
|
|
424
|
+
return [0, (Time.httpdate(value) - Time.now) * 1000].max
|
|
425
|
+
rescue ArgumentError
|
|
426
|
+
# An invalid header allows the documented JSON fallback.
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
value = parsed['retry_after'] if parsed.instance_of?(Hash)
|
|
430
|
+
return unless Validation.finite?(value) && value >= 0
|
|
431
|
+
[value.to_f, Float::MAX / 1000].min * 1000
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
def safe_message(value)
|
|
435
|
+
return 'Kompressr request failed.' unless value.instance_of?(String)
|
|
436
|
+
result = value.gsub(@key, '[redacted]').gsub(/Bearer\s+\S+/i, 'Bearer [redacted]')
|
|
437
|
+
result = result.gsub(%r{https?://[^\s<>"']+}i, '[URL redacted]')
|
|
438
|
+
result = result.gsub(/\b(?:api_key|password|token|secret|signature)\s*[=:]\s*\S+/i, '[credential redacted]')
|
|
439
|
+
result.gsub(/[\x00-\x1f\x7f]/, ' ')[0, 500]
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
def request(path, method: 'GET', body: nil, multipart: nil, upload: false, timeout: nil, cancel: nil,
|
|
443
|
+
asset_id: nil, recover_upload_id: false, &parse)
|
|
444
|
+
Validation.invalid('cancel must be a CancellationToken.') unless cancel.nil? || cancel.instance_of?(CancellationToken)
|
|
445
|
+
budget = Validation.seconds(timeout.nil? ? (upload ? @upload_timeout : @request_timeout) : timeout)
|
|
446
|
+
payload = multipart || (body.nil? ? nil : Body.new(JSON.generate(body)))
|
|
447
|
+
context = Context.new(budget, cancel, method != 'GET', asset_id)
|
|
448
|
+
uri = URI.parse(upload ? @upload_base_url : @api_base_url)
|
|
449
|
+
attempt = 0
|
|
450
|
+
loop do
|
|
451
|
+
context.check!
|
|
452
|
+
context.status = nil
|
|
453
|
+
context.retry_after_ms = nil
|
|
454
|
+
begin
|
|
455
|
+
parsed = nil
|
|
456
|
+
status = nil
|
|
457
|
+
Capacity.acquire(:connection, context)
|
|
458
|
+
begin
|
|
459
|
+
socket = Network.connect(uri, context)
|
|
460
|
+
wire = Wire.new(socket, context)
|
|
461
|
+
host = uri.host + (uri.port == (uri.scheme == 'https' ? 443 : 80) ? '' : ":#{uri.port}")
|
|
462
|
+
headers = "#{method} #{path} HTTP/1.1\r\nHost: #{host}\r\nAuthorization: Bearer #{@key}\r\nAccept: application/json\r\nAccept-Encoding: identity\r\nConnection: close\r\nUser-Agent: kompressr-ruby/#{VERSION}\r\n"
|
|
463
|
+
headers << "Content-Type: #{payload.content_type}\r\n" if payload
|
|
464
|
+
headers << "Content-Length: #{payload ? payload.length : 0}\r\n" if payload || method != 'GET'
|
|
465
|
+
wire.write(headers + "\r\n")
|
|
466
|
+
payload&.each do |chunk|
|
|
467
|
+
context.check!
|
|
468
|
+
break if wire.write(chunk, watch_response: true) == :early_response
|
|
469
|
+
end
|
|
470
|
+
begin
|
|
471
|
+
status, head, raw = wire.response
|
|
472
|
+
raw.force_encoding(Encoding::UTF_8)
|
|
473
|
+
Validation.bad('API returned invalid UTF-8 JSON.') unless raw.valid_encoding?
|
|
474
|
+
parsed = JSON.parse(raw, max_nesting: 100, allow_nan: false)
|
|
475
|
+
Validation.bad('API returned invalid Unicode JSON.') unless Validation.valid_json_strings?(parsed)
|
|
476
|
+
rescue JSON::ParserError, EncodingError
|
|
477
|
+
parsed = nil
|
|
478
|
+
Validation.bad('API returned a non-JSON response.') if context.status && context.status.between?(200, 299)
|
|
479
|
+
rescue Error => error
|
|
480
|
+
raise unless error.kind == 'invalid_response' && context.status && !context.status.between?(200, 299)
|
|
481
|
+
parsed = nil
|
|
482
|
+
end
|
|
483
|
+
status = context.status
|
|
484
|
+
Validation.bad unless status
|
|
485
|
+
context.retry_after_ms = self.class.retry_after(head, parsed) if head
|
|
486
|
+
context.check!
|
|
487
|
+
if recover_upload_id && status.between?(200, 299) && parsed.instance_of?(Hash) && parsed['success'] == true && parsed['data'].instance_of?(Hash)
|
|
488
|
+
candidate = parsed['data']['asset_id']
|
|
489
|
+
context.asset_id = candidate.to_i if Validation.finite?(candidate) && candidate.positive? && candidate <= Validation::MAX_SAFE && candidate == candidate.to_i
|
|
490
|
+
end
|
|
491
|
+
failed = parsed.instance_of?(Hash) && parsed['success'] == false
|
|
492
|
+
unless status.between?(200, 299) && !failed
|
|
493
|
+
data = parsed.instance_of?(Hash) ? parsed : {}
|
|
494
|
+
code = data['code']
|
|
495
|
+
code = 'API_ERROR' unless code.instance_of?(String) && /\A[A-Z][A-Z0-9_]{0,99}\z/.match?(code) && !code.include?(@key)
|
|
496
|
+
raise Error.new(safe_message(data['error'] || "Kompressr returned HTTP #{status}."), kind: 'api', code: code,
|
|
497
|
+
status: status, retry_after_ms: context.retry_after_ms, asset_id: context.asset_id,
|
|
498
|
+
may_have_succeeded: method != 'GET' && (status < 400 || status >= 500)), cause: nil
|
|
499
|
+
end
|
|
500
|
+
result = parse.call(parsed)
|
|
501
|
+
context.check!
|
|
502
|
+
return result
|
|
503
|
+
ensure
|
|
504
|
+
socket.close if socket && !socket.closed?
|
|
505
|
+
Capacity.release(:connection)
|
|
506
|
+
end
|
|
507
|
+
rescue Error, SystemCallError, IOError, SocketError, OpenSSL::SSL::SSLError, PhaseTimeout => error
|
|
508
|
+
failure = if error.instance_of?(Error)
|
|
509
|
+
error
|
|
510
|
+
else
|
|
511
|
+
context.check!
|
|
512
|
+
context.error('network')
|
|
513
|
+
end
|
|
514
|
+
if failure.kind == 'invalid_response'
|
|
515
|
+
failure = Error.new(failure.message, kind: failure.kind, code: failure.code, status: context.status,
|
|
516
|
+
retry_after_ms: context.retry_after_ms, asset_id: context.asset_id,
|
|
517
|
+
may_have_succeeded: method != 'GET' && context.dispatched)
|
|
518
|
+
end
|
|
519
|
+
transient = failure.kind == 'network' || (failure.kind == 'api' && [429, 502, 503, 504].include?(failure.status))
|
|
520
|
+
raise failure, cause: nil unless method == 'GET' && transient && attempt < @retries
|
|
521
|
+
context.pause([0.25 * (2**attempt) + rand * 0.1, (failure.retry_after_ms || 0) / 1000].max)
|
|
522
|
+
attempt += 1
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
end
|
|
528
|
+
end
|