monkrb 0.22.2 → 0.23.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,197 @@
1
+ require "net/http"
2
+
3
+ module Monk
4
+ module Storage
5
+ module Backends
6
+ # Any S3-compatible service (Hetzner, R2, B2, AWS, MinIO): Monk names
7
+ # no provider, only an endpoint and a region. A frozen value holding
8
+ # settings only; the HTTP client is built for each call, inside the
9
+ # Ractor making it, like Mail's SMTP and PG::Connection
10
+ # (docs/adr/0018-storage-s3-compatible-direct-upload.md).
11
+ #
12
+ # endpoint is host[:port]. tls: false only for a local server
13
+ # (endpoint=http://..., refused in staging and production).
14
+ # path_style: true puts the bucket in the path instead of the host
15
+ # name, which local servers need.
16
+ S3 = Data.define(:bucket, :endpoint, :region, :access_key, :secret_key, :path_style, :tls,
17
+ :open_timeout, :read_timeout,) do
18
+ def initialize(bucket:, endpoint:, region:, access_key:, secret_key:, path_style: false, tls: true,
19
+ open_timeout: 5, read_timeout: 30)
20
+ super
21
+ end
22
+
23
+ # Returns the key. body is a String, or an IO that responds to size,
24
+ # read and rewind: its SHA-256 is computed first and signed, then it
25
+ # is rewound and streamed, so it's read twice but never held in
26
+ # memory (docs/history/plan-storage.md decision 11).
27
+ def put(key, body, content_type: Input::DEFAULT_CONTENT_TYPE)
28
+ Key.validate!(key)
29
+ Input.content_type!(content_type)
30
+ Input.body!(body)
31
+
32
+ response = request("PUT", key, body: body, headers: { "content-type" => content_type })
33
+ return key if response.is_a?(Net::HTTPSuccess)
34
+
35
+ fail!("put", key, response)
36
+ end
37
+
38
+ # The bytes, as a frozen binary String. range: is a Range of byte
39
+ # offsets; one that starts past the end gives "" (S3's 416).
40
+ def get(key, range: nil)
41
+ Key.validate!(key)
42
+ headers = {}
43
+ if range
44
+ first, last = Input.byte_span(range)
45
+ headers["range"] = "bytes=#{first}-#{last}"
46
+ end
47
+
48
+ response = request("GET", key, headers: headers)
49
+ case response
50
+ when Net::HTTPSuccess then response.body.to_s.b.freeze
51
+ when Net::HTTPNotFound then raise NotFoundError, "no stored object #{key.inspect}"
52
+ when Net::HTTPRangeNotSatisfiable then "".b.freeze
53
+ else fail!("get", key, response)
54
+ end
55
+ end
56
+
57
+ # A Storage::Object, or nil when the key is missing. A bucket that
58
+ # doesn't grant ListBucket answers 403, not 404, for a missing key:
59
+ # that's a RequestError here, as the credentials need fixing.
60
+ def head(key)
61
+ Key.validate!(key)
62
+ response = request("HEAD", key)
63
+ return nil if response.is_a?(Net::HTTPNotFound)
64
+
65
+ fail!("head", key, response) unless response.is_a?(Net::HTTPSuccess)
66
+
67
+ Storage::Object.new(key: key, byte_size: response["content-length"].to_i,
68
+ content_type: response["content-type"], etag: response["etag"].to_s.delete('"'),)
69
+ end
70
+
71
+ # Idempotent: S3 answers 204 for a missing key too.
72
+ def delete(key)
73
+ Key.validate!(key)
74
+ response = request("DELETE", key)
75
+ return nil if response.is_a?(Net::HTTPSuccess)
76
+
77
+ fail!("delete", key, response)
78
+ end
79
+
80
+ # Returns to. The content type is copied with the bytes. S3 can
81
+ # answer 200 to a copy that failed, with an <Error> in the body, so
82
+ # the body is checked too.
83
+ def copy(from, to)
84
+ Key.validate!(from)
85
+ Key.validate!(to)
86
+ source = SigV4.encode_path("/#{bucket}/#{from}")
87
+
88
+ response = request("PUT", to, headers: { "x-amz-copy-source" => source })
89
+ raise NotFoundError, "no stored object #{from.inspect}" if response.is_a?(Net::HTTPNotFound)
90
+ return to if response.is_a?(Net::HTTPSuccess) && !response.body.to_s.include?("<Error>")
91
+
92
+ fail!("copy #{from} ->", to, response)
93
+ end
94
+
95
+ # A signed GET link, the same for every call within one
96
+ # cache_window: signed at the window's start, valid for the window
97
+ # plus expires_in (docs/history/plan-storage.md decision 9).
98
+ def url(key, expires_in:, cache_window:, now: Time.now.to_i)
99
+ Key.validate!(key)
100
+ Input.link_times!(expires_in: expires_in, cache_window: cache_window)
101
+ query = SigV4.presign(method: "GET", path: path_for(key), headers: { "host" => host },
102
+ expires_in: cache_window + expires_in, credentials: credentials, now: now - (now % cache_window),)
103
+ "#{base_url}#{SigV4.encode_path(path_for(key))}?#{query}"
104
+ end
105
+
106
+ # A signed PUT link for a browser, with Content-Type and
107
+ # Content-Length among its signed headers: the bucket refuses a
108
+ # body of another length or type.
109
+ def presigned_put(key, content_type:, byte_size:, expires_in:, now: Time.now.to_i)
110
+ Key.validate!(key)
111
+ Input.content_type!(content_type)
112
+ Input.byte_size!(byte_size)
113
+ headers = { "host" => host, "content-type" => content_type, "content-length" => byte_size.to_s }
114
+ query = SigV4.presign(method: "PUT", path: path_for(key), headers: headers, expires_in: expires_in,
115
+ credentials: credentials, now: now,)
116
+
117
+ Upload.new(url: "#{base_url}#{SigV4.encode_path(path_for(key))}?#{query}",
118
+ headers: { "Content-Type" => content_type, "Content-Length" => byte_size.to_s }.freeze,)
119
+ end
120
+
121
+ # Never print the secret key -- a backend ends up in logs and
122
+ # exception messages more easily than anyone intends.
123
+ def inspect
124
+ "#<#{self.class.name} s3://#{access_key}:***@#{bucket} #{tls ? "https" : "http"}://#{endpoint} " \
125
+ "region=#{region}#{" path_style" if path_style}>"
126
+ end
127
+ alias_method :to_s, :inspect
128
+
129
+ private
130
+
131
+ def credentials
132
+ { access_key: access_key, secret_key: secret_key, region: region }
133
+ end
134
+
135
+ # Virtual-hosted (bucket.endpoint/key) unless path_style
136
+ # (endpoint/bucket/key).
137
+ def host
138
+ path_style ? endpoint : "#{bucket}.#{endpoint}"
139
+ end
140
+
141
+ def path_for(key)
142
+ path_style ? "/#{bucket}/#{key}" : "/#{key}"
143
+ end
144
+
145
+ def base_url
146
+ "#{tls ? "https" : "http"}://#{host}"
147
+ end
148
+
149
+ # One signed request, on a connection opened for it in the calling
150
+ # Ractor. Network and TLS failures become a RequestError, the
151
+ # original as its #cause.
152
+ def request(method, key, body: nil, headers: {})
153
+ payload_hash = body ? Input.sha256(body) : SigV4::EMPTY_PAYLOAD
154
+ signed = headers.merge("host" => host, "x-amz-content-sha256" => payload_hash)
155
+ signed["content-length"] = body.size.to_s if body
156
+ path = path_for(key)
157
+ authorization = SigV4.sign_headers(method: method, path: path, headers: signed,
158
+ payload_hash: payload_hash, credentials: credentials, now: Time.now.to_i,)
159
+
160
+ http_request = Net::HTTPGenericRequest.new(method, !body.nil?, method != "HEAD", SigV4.encode_path(path),
161
+ signed.merge(authorization).merge("accept-encoding" => "identity"),)
162
+ attach(http_request, body) if body
163
+ connection.start { |http| http.request(http_request) }
164
+ rescue IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError, Net::HTTPBadResponse,
165
+ Net::ProtocolError => e
166
+ raise RequestError, "S3 #{method} #{key} via #{host} failed: #{e.class}: #{e.message}"
167
+ end
168
+
169
+ def attach(http_request, body)
170
+ if body.is_a?(String)
171
+ http_request.body = body
172
+ else
173
+ body.rewind
174
+ http_request.body_stream = body
175
+ end
176
+ end
177
+
178
+ def connection
179
+ http = Net::HTTP.new(host.sub(/:\d+\z/, ""), endpoint[/:(\d+)\z/, 1]&.to_i || (tls ? 443 : 80))
180
+ http.use_ssl = tls
181
+ http.open_timeout = open_timeout
182
+ http.read_timeout = read_timeout
183
+ http.write_timeout = read_timeout
184
+ http
185
+ end
186
+
187
+ def fail!(operation, key, response)
188
+ code = response.body.to_s[%r{<Code>([^<]*)</Code>}, 1]
189
+ message = response.body.to_s[%r{<Message>([^<]*)</Message>}, 1]
190
+ detail = [code, message].compact.join(": ")
191
+ raise RequestError,
192
+ "S3 #{operation} #{key} via #{host} failed: HTTP #{response.code}#{" #{detail}" unless detail.empty?}"
193
+ end
194
+ end
195
+ end
196
+ end
197
+ end
@@ -0,0 +1,34 @@
1
+ module Monk
2
+ module Storage
3
+ class NotConfiguredError < StandardError
4
+ end
5
+
6
+ class InvalidStorageUrlError < StandardError
7
+ end
8
+
9
+ class MissingStorageUrlError < StandardError
10
+ end
11
+
12
+ class MissingStorageSecretError < StandardError
13
+ end
14
+
15
+ # An ArgumentError: a key is always the caller's input.
16
+ class InvalidKeyError < ArgumentError
17
+ end
18
+
19
+ # get or copy of a key that isn't there.
20
+ class NotFoundError < StandardError
21
+ end
22
+
23
+ # A call that failed on the way to or from the backend -- network, TLS,
24
+ # the disk, or the bucket refusing it. The original exception is the
25
+ # #cause, the way Monk::Mail::DeliveryError works.
26
+ class RequestError < StandardError
27
+ end
28
+
29
+ # Bytes that must never be served as uploaded (SVG, which can carry
30
+ # scripts).
31
+ class UnsafeContentError < StandardError
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,83 @@
1
+ module Monk
2
+ module Storage
3
+ # Checks on the arguments both backends take, so they refuse the same
4
+ # input the same way. Keys have their own module (Key).
5
+ module Input
6
+ DEFAULT_CONTENT_TYPE = "application/octet-stream".freeze
7
+ CHUNK_BYTES = 64 * 1024
8
+ # SigV4's longest signed link: 7 days. A url is valid until the end
9
+ # of its cache window plus expires_in, so the two together stay
10
+ # under it.
11
+ MAX_LINK_SECONDS = 7 * 24 * 3600
12
+ # type/subtype with optional parameters, and nothing that could end a
13
+ # header line.
14
+ CONTENT_TYPE = %r{\A[a-z0-9][a-z0-9!\#$&^_.+-]*/[a-z0-9][a-z0-9!\#$&^_.+-]*(;[ \t]*[ -~]*)?\z}i
15
+
16
+ module_function
17
+
18
+ def content_type!(value)
19
+ return value if value.is_a?(String) && value.match?(CONTENT_TYPE)
20
+
21
+ raise ArgumentError, "content_type: must be a MIME type such as \"image/png\", not #{value.inspect}"
22
+ end
23
+
24
+ # A String, or an IO the backend can measure, read, and read again
25
+ # (S3 hashes the body before sending it).
26
+ def body!(body)
27
+ return body if body.is_a?(String)
28
+ return body if %i[size read rewind].all? { |name| body.respond_to?(name) }
29
+
30
+ raise ArgumentError,
31
+ "a stored body is a String or an IO that responds to size, read and rewind (File, Tempfile, " \
32
+ "StringIO), not #{body.class}; write a pipe to a Tempfile first"
33
+ end
34
+
35
+ def link_times!(expires_in:, cache_window:)
36
+ { expires_in: expires_in, cache_window: cache_window }.each do |name, value|
37
+ next if value.is_a?(Integer) && value.positive?
38
+
39
+ raise ArgumentError, "#{name}: is a number of seconds above 0, not #{value.inspect}"
40
+ end
41
+ return if expires_in + cache_window <= MAX_LINK_SECONDS
42
+
43
+ raise ArgumentError,
44
+ "expires_in: #{expires_in} + cache_window: #{cache_window} is more than 7 days (#{MAX_LINK_SECONDS}), " \
45
+ "the longest a signed S3 link can be valid"
46
+ end
47
+
48
+ def byte_size!(value)
49
+ return value if value.is_a?(Integer) && !value.negative?
50
+
51
+ raise ArgumentError, "byte_size: is the upload's length in bytes, not #{value.inspect}"
52
+ end
53
+
54
+ # The body's SHA-256 in hex, read in chunks and rewound, so an IO can
55
+ # then be sent from the start.
56
+ def sha256(body)
57
+ return Digest::SHA256.hexdigest(body) if body.is_a?(String)
58
+
59
+ digest = Digest::SHA256.new
60
+ body.rewind
61
+ while (chunk = body.read(CHUNK_BYTES))
62
+ digest << chunk
63
+ end
64
+ body.rewind
65
+ digest.hexdigest
66
+ end
67
+
68
+ # A Range of byte offsets -> [first, last], last nil for "to the end".
69
+ def byte_span(range)
70
+ first = range.begin if range.is_a?(Range)
71
+ last = range.end if range.is_a?(Range)
72
+ unless first.is_a?(Integer) && first >= 0 && (last.nil? || last.is_a?(Integer))
73
+ raise ArgumentError, "range: is a Range of byte offsets from 0, e.g. 0..15, not #{range.inspect}"
74
+ end
75
+
76
+ last -= 1 if last && range.exclude_end?
77
+ raise ArgumentError, "range: #{range.inspect} is empty" if last && last < first
78
+
79
+ [first, last]
80
+ end
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,40 @@
1
+ require "securerandom"
2
+
3
+ module Monk
4
+ module Storage
5
+ # Keys name objects in the bucket (or under the file backend's root).
6
+ # Monk makes them random and never from a user's filename, which rules
7
+ # out path traversal and keeps file names out of URLs
8
+ # (docs/adr/0018-storage-s3-compatible-direct-upload.md).
9
+ module Key
10
+ # Lowercase only: macOS disks are case-insensitive, and the file
11
+ # backend must not merge two keys a bucket keeps apart. A segment
12
+ # can't start with a dot, so no "." or ".." segment, and no empty one.
13
+ FORMAT = %r{\A[a-z0-9][a-z0-9._-]*(/[a-z0-9][a-z0-9._-]*)*\z}
14
+ MAX_BYTES = 512
15
+
16
+ module_function
17
+
18
+ # "tmp/1f0c...": 128 random bits under prefix.
19
+ def generate(prefix:)
20
+ validate!(prefix)
21
+
22
+ "#{prefix}/#{SecureRandom.hex(16)}"
23
+ end
24
+
25
+ # Runs on every call to either backend, so a key that would mean
26
+ # something else on disk than in a bucket never reaches one.
27
+ def validate!(key)
28
+ raise InvalidKeyError, "a storage key is a String, not #{key.class}" unless key.is_a?(String)
29
+ raise InvalidKeyError, "storage key is #{key.bytesize} bytes; at most #{MAX_BYTES}" if key.bytesize > MAX_BYTES
30
+ unless key.match?(FORMAT)
31
+ raise InvalidKeyError,
32
+ "storage key #{key.inspect}: segments of a-z, 0-9, . _ and -, each starting with a letter or digit, " \
33
+ "joined by / (e.g. from Monk::Storage.generate_key(prefix: \"tmp\"))"
34
+ end
35
+
36
+ key
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,7 @@
1
+ module Monk
2
+ module Storage
3
+ # What head returns for a stored object. etag is the bytes' MD5 in hex,
4
+ # without the quotes S3 puts around it.
5
+ Object = Data.define(:key, :byte_size, :content_type, :etag)
6
+ end
7
+ end
@@ -0,0 +1,110 @@
1
+ module Monk
2
+ module Storage
3
+ ROUTE_PREFIX = "/_monk/storage/".freeze
4
+
5
+ # PUT and GET /_monk/storage/<token>: the file backend standing in for
6
+ # a bucket's signed URLs, refusing what a bucket would refuse, the way
7
+ # it would (docs/history/plan-storage.md decisions 7 and 8). Served whenever
8
+ # the backend is file://, in any environment, since its links only
9
+ # work if the routes exist.
10
+ module Routes
11
+ # Shareable, not only frozen: worker Ractors serve these routes.
12
+ NOSNIFF = Ractor.make_shareable({ "x-content-type-options" => "nosniff" })
13
+ RANGE = /\Abytes=(\d*)-(\d*)\z/
14
+
15
+ module_function
16
+
17
+ # A Rack response, or nil when the request isn't for these routes.
18
+ def response(env, backend, now: Time.now.to_i)
19
+ path = env["PATH_INFO"].to_s
20
+ return nil unless backend.is_a?(Backends::File) && path.start_with?(ROUTE_PREFIX)
21
+
22
+ payload = Token.verify(path.delete_prefix(ROUTE_PREFIX), backend.secret)
23
+ return refuse(403, "signature doesn't match") unless payload
24
+ return refuse(403, "link expired") unless payload[:e].is_a?(Integer) && payload[:e] > now
25
+ return refuse(403, "this link is for #{payload[:m]}") unless env["REQUEST_METHOD"] == payload[:m]
26
+
27
+ case payload[:m]
28
+ when "PUT" then upload(env, backend, payload)
29
+ when "GET" then download(env, backend, payload, now)
30
+ else refuse(403, "signature doesn't match")
31
+ end
32
+ rescue InvalidKeyError
33
+ refuse(403, "signature doesn't match")
34
+ end
35
+
36
+ # The browser's PUT. Content-Length and Content-Type are inside the
37
+ # token, as in a presigned S3 PUT: another value is a 403. A body
38
+ # shorter or longer than its Content-Length is a 400.
39
+ def upload(env, backend, payload)
40
+ return refuse(403, "Content-Length isn't the signed one") unless env["CONTENT_LENGTH"] == payload[:l].to_s
41
+ return refuse(403, "Content-Type isn't the signed one") unless env["CONTENT_TYPE"] == payload[:t]
42
+
43
+ md5 = backend.receive(payload[:k], env["rack.input"], content_type: payload[:t], byte_size: payload[:l])
44
+ return refuse(400, "the body isn't Content-Length bytes") unless md5
45
+
46
+ [200, NOSNIFF.merge("etag" => %("#{md5}")), []]
47
+ end
48
+
49
+ # The signed GET: the stored bytes, as their stored type, cached
50
+ # privately until the link expires. A Range is answered with 206, as
51
+ # a bucket does: Safari won't play <audio> or <video> without it.
52
+ def download(env, backend, payload, now)
53
+ object = backend.head(payload[:k])
54
+ return refuse(404, "no such object") unless object
55
+
56
+ headers = NOSNIFF.merge(
57
+ "content-type" => object.content_type,
58
+ "accept-ranges" => "bytes",
59
+ "etag" => %("#{object.etag}"),
60
+ "cache-control" => "private, max-age=#{payload[:e] - now}",
61
+ )
62
+ span = span(env["HTTP_RANGE"], object.byte_size)
63
+ return whole(backend, payload[:k], headers) if span.nil?
64
+ return refuse(416, "range not satisfiable", "content-range" => "bytes */#{object.byte_size}") if span == :none
65
+
66
+ first, last = span
67
+ bytes = backend.get(payload[:k], range: first..last)
68
+ [206, headers.merge("content-range" => "bytes #{first}-#{last}/#{object.byte_size}",
69
+ "content-length" => bytes.bytesize.to_s,), [bytes],]
70
+ rescue NotFoundError
71
+ refuse(404, "no such object")
72
+ end
73
+
74
+ def whole(backend, key, headers)
75
+ bytes = backend.get(key)
76
+ [200, headers.merge("content-length" => bytes.bytesize.to_s), [bytes]]
77
+ end
78
+
79
+ # One range of a Range header -> [first, last] within size; :none
80
+ # when it can't be satisfied; nil to ignore the header and send the
81
+ # whole object (no header, a syntax this doesn't read, several
82
+ # ranges), as RFC 9110 allows.
83
+ def span(header, size)
84
+ match = header && RANGE.match(header)
85
+ return nil unless match
86
+
87
+ first, last = match.captures
88
+ return nil if first.empty? && last.empty?
89
+
90
+ if first.empty?
91
+ suffix = last.to_i
92
+ return :none if suffix.zero? || size.zero?
93
+
94
+ return [[size - suffix, 0].max, size - 1]
95
+ end
96
+
97
+ first = first.to_i
98
+ last = last.empty? ? size - 1 : [last.to_i, size - 1].min
99
+ return nil if last < first && first < size
100
+ return :none if first >= size
101
+
102
+ [first, last]
103
+ end
104
+
105
+ def refuse(status, reason, extra = {})
106
+ [status, NOSNIFF.merge("content-type" => "text/plain; charset=utf-8").merge(extra), ["#{reason}\n"]]
107
+ end
108
+ end
109
+ end
110
+ end
@@ -0,0 +1,124 @@
1
+ require "digest"
2
+ # Loaded here, at boot: left to Digest.const_missing it loads on the first
3
+ # signed request, and a worker Ractor signing another in parallel can
4
+ # catch it half-defined (test/digest_preload_test.rb).
5
+ require "digest/sha2"
6
+ require "openssl"
7
+
8
+ module Monk
9
+ module Storage
10
+ # AWS Signature Version 4, for the S3 backend: the Authorization header
11
+ # of a request Monk sends, and the query string of a link a browser
12
+ # uses (a presigned PUT, a signed GET). Pure functions of their
13
+ # arguments, the time included, so they're tested against AWS's
14
+ # published vectors without stubbing Time (docs/history/plan-storage.md
15
+ # decision 10).
16
+ #
17
+ # Paths and query strings are encoded the S3 way: each byte outside
18
+ # A-Z a-z 0-9 - _ . ~ as %XX, "/" kept in paths, and paths encoded once
19
+ # and never normalized (S3 differs from other AWS services there).
20
+ module SigV4
21
+ ALGORITHM = "AWS4-HMAC-SHA256".freeze
22
+ # The payload hash of a presigned link: the browser sends the body,
23
+ # so it can't be signed in advance.
24
+ UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD".freeze
25
+ EMPTY_PAYLOAD = Digest::SHA256.hexdigest("").freeze
26
+ MAX_EXPIRES = 7 * 24 * 3600
27
+ UNRESERVED = /[^A-Za-z0-9\-_.~]/
28
+
29
+ module_function
30
+
31
+ # The headers to add to a request: x-amz-date and authorization.
32
+ # headers: are the ones to sign, host included (and
33
+ # x-amz-content-sha256 for S3); their names may be in any case.
34
+ # payload_hash: is the body's SHA-256 in hex. now: is a Unix time.
35
+ def sign_headers(method:, path:, headers:, payload_hash:, credentials:, now:, query: {})
36
+ date = timestamp(now)
37
+ signed = canonical_headers(headers.merge("x-amz-date" => date))
38
+ signature = signature(method, path, canonical_query(query), signed, payload_hash, credentials, date)
39
+ {
40
+ "x-amz-date" => date,
41
+ "authorization" => "#{ALGORITHM} Credential=#{credentials[:access_key]}/#{scope(credentials, date)}, " \
42
+ "SignedHeaders=#{signed.keys.join(";")}, Signature=#{signature}",
43
+ }
44
+ end
45
+
46
+ # The query string of a presigned link, signature last: the request's
47
+ # own query: parameters, then the X-Amz-* ones. headers: are the ones
48
+ # the client must send exactly (host, and for a presigned PUT
49
+ # content-length and content-type).
50
+ def presign(method:, path:, headers:, expires_in:, credentials:, now:, query: {},
51
+ payload_hash: UNSIGNED_PAYLOAD)
52
+ unless expires_in.is_a?(Integer) && expires_in.between?(1, MAX_EXPIRES)
53
+ raise ArgumentError, "a presigned link is valid 1 to #{MAX_EXPIRES} seconds, not #{expires_in.inspect}"
54
+ end
55
+
56
+ date = timestamp(now)
57
+ signed = canonical_headers(headers)
58
+ params = query.merge(
59
+ "X-Amz-Algorithm" => ALGORITHM,
60
+ "X-Amz-Credential" => "#{credentials[:access_key]}/#{scope(credentials, date)}",
61
+ "X-Amz-Date" => date,
62
+ "X-Amz-Expires" => expires_in.to_s,
63
+ "X-Amz-SignedHeaders" => signed.keys.join(";"),
64
+ )
65
+ canonical = canonical_query(params)
66
+ "#{canonical}&X-Amz-Signature=#{signature(method, path, canonical, signed, payload_hash, credentials, date)}"
67
+ end
68
+
69
+ def signature(method, path, query, headers, payload_hash, credentials, date)
70
+ request = canonical_request(method, path, query, headers, payload_hash)
71
+ OpenSSL::HMAC.hexdigest("SHA256", signing_key(credentials, date), string_to_sign(request, credentials, date))
72
+ end
73
+
74
+ def canonical_request(method, path, query, headers, payload_hash)
75
+ [
76
+ method,
77
+ encode_path(path),
78
+ query,
79
+ headers.map { |name, value| "#{name}:#{value}\n" }.join,
80
+ headers.keys.join(";"),
81
+ payload_hash,
82
+ ].join("\n")
83
+ end
84
+
85
+ def string_to_sign(canonical_request, credentials, date)
86
+ [ALGORITHM, date, scope(credentials, date), Digest::SHA256.hexdigest(canonical_request)].join("\n")
87
+ end
88
+
89
+ def signing_key(credentials, date)
90
+ key = "AWS4#{credentials[:secret_key]}"
91
+ [date[0, 8], credentials[:region], credentials.fetch(:service, "s3"), "aws4_request"].each do |part|
92
+ key = OpenSSL::HMAC.digest("SHA256", key, part)
93
+ end
94
+ key
95
+ end
96
+
97
+ def scope(credentials, date)
98
+ "#{date[0, 8]}/#{credentials[:region]}/#{credentials.fetch(:service, "s3")}/aws4_request"
99
+ end
100
+
101
+ # Lowercase names, sorted; values trimmed, inner runs of spaces made one.
102
+ def canonical_headers(headers)
103
+ headers.to_h { |name, value| [name.to_s.downcase, value.to_s.strip.gsub(/\s+/, " ")] }.sort.to_h
104
+ end
105
+
106
+ def canonical_query(params)
107
+ params.map { |name, value| [encode(name.to_s), encode(value.to_s)] }.sort
108
+ .map { |name, value| "#{name}=#{value}" }.join("&")
109
+ end
110
+
111
+ def encode_path(path)
112
+ path.split("/", -1).map { |segment| encode(segment) }.join("/")
113
+ end
114
+
115
+ def encode(text)
116
+ text.b.gsub(UNRESERVED) { |byte| format("%%%02X", byte.ord) }
117
+ end
118
+
119
+ def timestamp(now)
120
+ Time.at(now).utc.strftime("%Y%m%dT%H%M%SZ")
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,81 @@
1
+ module Monk
2
+ module Storage
3
+ # A file's type from its first bytes, since the Content-Type a client
4
+ # sends can't be trusted. Defence in depth: the type a signed link
5
+ # serves is the one the app signed into presigned_put from its allowed
6
+ # list, and this catches a file whose bytes don't match it
7
+ # (docs/history/plan-storage.md decision 12). At confirm, read only these
8
+ # bytes:
9
+ #
10
+ # head = Monk::Storage.get(key, range: Monk::Storage::Sniff::RANGE)
11
+ # Monk::Storage::Sniff.refuse_svg!(head)
12
+ # Monk::Storage::Sniff.content_type(head) # => "image/png" or nil
13
+ module Sniff
14
+ # 64 bytes, not 16: an Ogg page's header is 27 bytes plus one byte
15
+ # per segment, so OpusHead starts at byte 28 on a usual first page.
16
+ RANGE = 0..63
17
+
18
+ PNG = "\x89PNG\r\n\x1A\n".b.freeze
19
+ JPEG = "\xFF\xD8\xFF".b.freeze
20
+ GIFS = Ractor.make_shareable(["GIF87a".b, "GIF89a".b])
21
+ PDF = "%PDF-".b.freeze
22
+ # Where an Ogg page's segment count sits; the segment table follows.
23
+ OGG_SEGMENTS_AT = 26
24
+ # Lowercased before matching, so <SVG and <?XML count too.
25
+ SVG_MARKERS = Ractor.make_shareable(["<svg".b, "<?xml".b])
26
+
27
+ module_function
28
+
29
+ # "image/png", "image/jpeg", "image/gif", "image/webp", "audio/ogg"
30
+ # (an Ogg stream whose first packet is OpusHead), "application/pdf",
31
+ # or nil for anything else, an empty String included.
32
+ def content_type(bytes)
33
+ head = prefix(bytes)
34
+
35
+ if head.start_with?(PNG) then "image/png"
36
+ elsif head.start_with?(JPEG) then "image/jpeg"
37
+ elsif head.start_with?(*GIFS) then "image/gif"
38
+ elsif webp?(head) then "image/webp"
39
+ elsif opus?(head) then "audio/ogg"
40
+ elsif head.start_with?(PDF) then "application/pdf"
41
+ end
42
+ end
43
+
44
+ # Raises UnsafeContentError when <svg or <?xml appears in the first
45
+ # bytes: SVG can carry scripts, and an SVG may start with an XML
46
+ # declaration, a comment or a doctype before its <svg. Only RANGE is
47
+ # looked at, so the XMP metadata deep inside a JPEG or PDF doesn't
48
+ # count. Returns bytes.
49
+ def refuse_svg!(bytes)
50
+ head = prefix(bytes).downcase
51
+ if SVG_MARKERS.any? { |marker| head.include?(marker) }
52
+ raise UnsafeContentError, "the file starts like SVG or XML, which can carry scripts; refused"
53
+ end
54
+
55
+ bytes
56
+ end
57
+
58
+ def prefix(bytes)
59
+ unless bytes.is_a?(String)
60
+ raise ArgumentError,
61
+ "Sniff takes the file's first bytes as a String, not #{bytes.class}"
62
+ end
63
+
64
+ bytes.byteslice(RANGE).b
65
+ end
66
+
67
+ # RIFF, a 4-byte size, then WEBP.
68
+ def webp?(head)
69
+ head.start_with?("RIFF") && head.byteslice(8, 4) == "WEBP"
70
+ end
71
+
72
+ def opus?(head)
73
+ return false unless head.start_with?("OggS") && head.bytesize > OGG_SEGMENTS_AT
74
+
75
+ packet_at = OGG_SEGMENTS_AT + 1 + head.getbyte(OGG_SEGMENTS_AT)
76
+ head.byteslice(packet_at, 8) == "OpusHead"
77
+ end
78
+ private_class_method :prefix, :webp?, :opus?
79
+ end
80
+ end
81
+ end