serve_byte_range 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +31 -7
- data/.gitignore +3 -2
- data/.standard.yml +4 -0
- data/README.md +58 -0
- data/Rakefile +19 -6
- data/Steepfile +10 -0
- data/gemfiles/rack_2.gemfile +5 -0
- data/gemfiles/rack_3.gemfile +5 -0
- data/gemfiles/typecheck.gemfile +7 -0
- data/lib/serve_byte_range/sequence.rb +152 -0
- data/lib/serve_byte_range/version.rb +5 -1
- data/lib/serve_byte_range.rb +222 -43
- data/serve_byte_range.gemspec +9 -4
- data/sig/serve_byte_range/sequence.rbs +91 -0
- data/sig/serve_byte_range/version.rbs +5 -0
- data/sig/serve_byte_range.rbs +256 -0
- data/sig_shims/rack.rbs +9 -0
- data/test/byte_comparator_test.rb +46 -0
- data/test/range_server_test.rb +123 -6
- data/test/sequence_test.rb +224 -0
- data/test/support/byte_comparator.rb +47 -0
- data/test/support/bytes_assertions.rb +10 -0
- metadata +36 -48
- data/rbi/serve_byte_range.rbi +0 -208
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b89324c74c04f679e18acdf0184acb3f82b7b65e96b27bae9216b82ace69840d
|
|
4
|
+
data.tar.gz: e302baff4f9c8cb95569fc9cca6b468526e821ace7a82ab16c4574c3dd75bb33
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 4ade7e302976c27c11d19d5541e4e0b2a524011f60091fa6ad6b7439882cd3b7ece4515c2791febe7f47de7a9a1bd1d591161a2e09ec40290489cdc4e0f4e433
|
|
7
|
+
data.tar.gz: 285b227ea76fff572f399a2492d2e7ef02d89a178a7c793a78568627bb16b408fccddead4417d6cd1fbc70a37f0c1eedbbe23c23a6b821e32ba70e9424123bf0
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -7,24 +7,44 @@ env:
|
|
|
7
7
|
BUNDLE_PATH: vendor/bundle
|
|
8
8
|
|
|
9
9
|
jobs:
|
|
10
|
-
|
|
11
|
-
name: Tests
|
|
10
|
+
test:
|
|
11
|
+
name: Tests (Ruby ${{ matrix.ruby }}, Rack ${{ matrix.rack }})
|
|
12
12
|
runs-on: ubuntu-22.04
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
ruby: ["2.7", "3.4"]
|
|
17
|
+
rack: ["2", "3"]
|
|
18
|
+
env:
|
|
19
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/rack_${{ matrix.rack }}.gemfile
|
|
13
20
|
steps:
|
|
14
21
|
- name: Checkout
|
|
15
22
|
uses: actions/checkout@v4
|
|
16
23
|
- name: Setup Ruby
|
|
17
24
|
uses: ruby/setup-ruby@v1
|
|
18
25
|
with:
|
|
19
|
-
ruby-version:
|
|
26
|
+
ruby-version: ${{ matrix.ruby }}
|
|
20
27
|
bundler-cache: true
|
|
21
28
|
- name: "Tests"
|
|
22
29
|
run: bundle exec rake test --backtrace
|
|
30
|
+
lint:
|
|
31
|
+
name: Lint (2.7)
|
|
32
|
+
runs-on: ubuntu-22.04
|
|
33
|
+
steps:
|
|
34
|
+
- name: Checkout
|
|
35
|
+
uses: actions/checkout@v4
|
|
36
|
+
- name: Setup Ruby
|
|
37
|
+
uses: ruby/setup-ruby@v1
|
|
38
|
+
with:
|
|
39
|
+
ruby-version: 2.7
|
|
40
|
+
bundler-cache: true
|
|
23
41
|
- name: "Lint"
|
|
24
42
|
run: bundle exec rake standard
|
|
25
|
-
|
|
26
|
-
name:
|
|
43
|
+
typecheck:
|
|
44
|
+
name: Typecheck (3.4)
|
|
27
45
|
runs-on: ubuntu-22.04
|
|
46
|
+
env:
|
|
47
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/typecheck.gemfile
|
|
28
48
|
steps:
|
|
29
49
|
- name: Checkout
|
|
30
50
|
uses: actions/checkout@v4
|
|
@@ -33,5 +53,9 @@ jobs:
|
|
|
33
53
|
with:
|
|
34
54
|
ruby-version: 3.4
|
|
35
55
|
bundler-cache: true
|
|
36
|
-
- name: "
|
|
37
|
-
run: bundle exec rake
|
|
56
|
+
- name: "Steep"
|
|
57
|
+
run: bundle exec rake steep
|
|
58
|
+
- name: "Signatures in sig/ are up to date"
|
|
59
|
+
run: |
|
|
60
|
+
git add --intent-to-add sig
|
|
61
|
+
git diff --exit-code sig || (echo "Run 'rake rbs' and commit the changes in sig/" && exit 1)
|
data/.gitignore
CHANGED
data/.standard.yml
CHANGED
data/README.md
CHANGED
|
@@ -28,3 +28,61 @@ You can also retrieve the `range_in_file` from an external resource, you can als
|
|
|
28
28
|
* Will be correctly pre-sized with `Content-Length`
|
|
29
29
|
* Will not be generated and buffered eagerly, but will get generated as you serve the content out
|
|
30
30
|
|
|
31
|
+
Every string you `write` gets copied before it is passed on to the webserver, since `IO.copy_stream` and `IO#readpartial` with a buffer reuse the same `String` for every chunk, and middleware like `Rack::ETag` holds on to the chunks. Strings which are frozen and already in the `BINARY` encoding are passed on as-is, as nobody can change them afterwards.
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
## Serving a resource made of several segments
|
|
35
|
+
|
|
36
|
+
If your response is glued together from several pieces - byte ranges of objects in S3, literal strings and so on - use a `ServeByteRange::Sequence`. A segment can be any object, since you give its size explicitly. For every requested range you get the segments it touches, along with the range of bytes inside each segment:
|
|
37
|
+
|
|
38
|
+
```ruby
|
|
39
|
+
header_file = File.open("header.bin", "rb")
|
|
40
|
+
|
|
41
|
+
seq = ServeByteRange::Sequence.new
|
|
42
|
+
seq.add_segment(header_file, size: header_file.size, etag: "#{header_file.mtime.to_i}-#{header_file.size}")
|
|
43
|
+
seq.add_segment(s3_object, size: s3_object.content_length, etag: s3_object.etag)
|
|
44
|
+
seq << "--- redacted ---\n" # Strings are sized in bytes, and their contents go into the ETag
|
|
45
|
+
seq.add_segment(other_s3_object, size: other_s3_object.content_length, etag: other_s3_object.etag)
|
|
46
|
+
|
|
47
|
+
status, headers, body = ServeByteRange.serve_sequence(env, seq) do |segment, range_in_segment, io|
|
|
48
|
+
case segment
|
|
49
|
+
when String
|
|
50
|
+
io.write(segment.byteslice(range_in_segment))
|
|
51
|
+
when File
|
|
52
|
+
# Copies range_in_segment.size bytes starting at range_in_segment.begin, without moving the file position
|
|
53
|
+
IO.copy_stream(segment, io, range_in_segment.size, range_in_segment.begin)
|
|
54
|
+
when Aws::S3::Object
|
|
55
|
+
# Passing a block to `get` streams the body in chunks instead of buffering it in memory
|
|
56
|
+
segment.get(range: "bytes=%d-%d" % [range_in_segment.begin, range_in_segment.end]) do |chunk|
|
|
57
|
+
io.write(chunk)
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# The webserver calls `close` on the body once the response is sent, or if the client goes away
|
|
63
|
+
[status, headers, Rack::BodyProxy.new(body) { header_file.close }]
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The body is generated lazily while the webserver sends the response, so the `File` has to stay open until then. Closing it right after `serve_sequence` returns would break the response, and not closing it at all leaks a file descriptor per request - hence the `Rack::BodyProxy`.
|
|
67
|
+
|
|
68
|
+
The `ETag` of the response is derived from the ETags of the segments. If you do not give an `etag:` for a segment, a change which keeps its size the same will not be noticed, so the `ETag` of the `Sequence` becomes weak (`W/"..."`). Conditional GETs keep working with a weak `ETag`, but `If-Range` won't match it, so clients resuming a download get the whole response again instead of stitching together bytes from two different versions. Sequences can be nested. Segments can't change once added: Strings get copied and frozen, and a `Sequence` gets frozen when it goes into another one. If a segment writes more or fewer bytes than its range requires, you get an exception saying which segment it was.
|
|
69
|
+
|
|
70
|
+
## Development
|
|
71
|
+
|
|
72
|
+
Range parsing is delegated to Rack, and its behavior differs between Rack versions. To run the tests against a specific major version of Rack:
|
|
73
|
+
|
|
74
|
+
```shell
|
|
75
|
+
BUNDLE_GEMFILE=gemfiles/rack_2.gemfile bundle install
|
|
76
|
+
BUNDLE_GEMFILE=gemfiles/rack_2.gemfile bundle exec rake test
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
CI runs the tests against both `gemfiles/rack_2.gemfile` and `gemfiles/rack_3.gemfile`.
|
|
80
|
+
|
|
81
|
+
Types are written inline as [rbs-inline](https://github.com/soutaro/rbs-inline) annotations, and the RBS signatures in `sig/` are generated from them - so after changing a signature, regenerate them and typecheck with Steep. rbs-inline and Steep need a recent Ruby, so they live in a Gemfile of their own:
|
|
82
|
+
|
|
83
|
+
```shell
|
|
84
|
+
BUNDLE_GEMFILE=gemfiles/typecheck.gemfile bundle install
|
|
85
|
+
bundle exec rake steep # runs `rake rbs` first
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
API docs are RDoc with Markdown markup, `bundle exec rake rdoc` writes them into `doc/`.
|
data/Rakefile
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "bundler/gem_tasks"
|
|
4
4
|
require "rake/testtask"
|
|
5
5
|
require "standard/rake"
|
|
6
|
+
require "rdoc/task"
|
|
6
7
|
|
|
7
8
|
Rake::TestTask.new(:test) do |t|
|
|
8
9
|
t.libs << "test"
|
|
@@ -15,12 +16,24 @@ task :format do
|
|
|
15
16
|
`bundle exec magic_frozen_string_literal .`
|
|
16
17
|
end
|
|
17
18
|
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
RDoc::Task.new(:rdoc) do |rd|
|
|
20
|
+
rd.rdoc_dir = "doc"
|
|
21
|
+
rd.main = "README.md"
|
|
22
|
+
rd.markup = "markdown"
|
|
23
|
+
rd.rdoc_files.include("README.md", "lib/**/*.rb")
|
|
20
24
|
end
|
|
21
25
|
|
|
22
|
-
#
|
|
23
|
-
|
|
24
|
-
Rake::Task["build"].enhance(["generate_typedefs"])
|
|
26
|
+
# rbs-inline and steep need Ruby 3.1+, so they live in a Gemfile of their own
|
|
27
|
+
TYPECHECK_ENV = {"BUNDLE_GEMFILE" => File.expand_path("gemfiles/typecheck.gemfile", __dir__)}
|
|
25
28
|
|
|
26
|
-
|
|
29
|
+
desc "Generate RBS signatures in sig/ from the inline annotations in lib/"
|
|
30
|
+
task :rbs do
|
|
31
|
+
sh TYPECHECK_ENV, "bundle exec rbs-inline --output=sig lib"
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
desc "Typecheck with Steep"
|
|
35
|
+
task steep: :rbs do
|
|
36
|
+
sh TYPECHECK_ENV, "bundle exec steep check"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
task default: [:test, :standard]
|
data/Steepfile
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
D = Steep::Diagnostic
|
|
2
|
+
|
|
3
|
+
target :lib do
|
|
4
|
+
signature "sig"
|
|
5
|
+
# Kept out of sig/ so that it does not ship with the gem and clash with Rack signatures apps may have
|
|
6
|
+
signature "sig_shims"
|
|
7
|
+
check "lib"
|
|
8
|
+
library "digest"
|
|
9
|
+
configure_code_diagnostics(D::Ruby.default)
|
|
10
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# rbs_inline: enabled
|
|
4
|
+
|
|
5
|
+
require "digest"
|
|
6
|
+
|
|
7
|
+
module ServeByteRange
|
|
8
|
+
# A resource composed of several segments glued together - for example, byte ranges of objects
|
|
9
|
+
# in remote storage interleaved with literal strings. A segment can be any object, since its
|
|
10
|
+
# size is given explicitly. A Sequence can also be a segment of another Sequence.
|
|
11
|
+
#
|
|
12
|
+
# seq = ServeByteRange::Sequence.new
|
|
13
|
+
# seq.add_segment(s3_object, size: 1024, etag: s3_object.etag)
|
|
14
|
+
# seq << "Literal text"
|
|
15
|
+
# ServeByteRange.serve_sequence(env, seq) do |segment, range_in_segment, io|
|
|
16
|
+
# io.write(segment.is_a?(String) ? segment.byteslice(range_in_segment) : fetch_range(segment, range_in_segment))
|
|
17
|
+
# end
|
|
18
|
+
class Sequence
|
|
19
|
+
Interval = Struct.new(
|
|
20
|
+
:segment, #: untyped
|
|
21
|
+
:size, #: Integer
|
|
22
|
+
:offset, #: Integer
|
|
23
|
+
:etag, #: String
|
|
24
|
+
:weak #: bool
|
|
25
|
+
)
|
|
26
|
+
private_constant :Interval
|
|
27
|
+
|
|
28
|
+
# @rbs @intervals: Array[Interval]
|
|
29
|
+
# @rbs @etag: String?
|
|
30
|
+
|
|
31
|
+
# The total size of all segments in bytes
|
|
32
|
+
attr_reader :size #: Integer
|
|
33
|
+
|
|
34
|
+
# @rbs *segments: untyped -- segments to add with `<<`
|
|
35
|
+
def initialize(*segments)
|
|
36
|
+
@intervals = []
|
|
37
|
+
@size = 0
|
|
38
|
+
segments.each { |segment| self << segment }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Adds a segment of a given size. Segments of zero size are skipped. The size and ETag are taken
|
|
42
|
+
# as they are now, so a String gets copied and frozen, and a Sequence gets frozen when added.
|
|
43
|
+
#
|
|
44
|
+
# The `etag` goes into the ETag of the Sequence. Without it only the size is known, which won't catch
|
|
45
|
+
# content changes that keep the size the same - so the ETag of the Sequence becomes weak. It can be
|
|
46
|
+
# quoted (as S3 returns it) or not, and a weak one (`W/"..."`) makes the ETag of the Sequence weak too.
|
|
47
|
+
#
|
|
48
|
+
# @rbs segment: untyped -- any object, it gets yielded back when serving
|
|
49
|
+
# @rbs size: Integer -- the size of the segment in bytes
|
|
50
|
+
# @rbs etag: untyped -- a validator for the segment
|
|
51
|
+
# @rbs return: self
|
|
52
|
+
def add_segment(segment, size:, etag: nil)
|
|
53
|
+
raise ArgumentError, "Segment size must be an Integer >= 0, but was #{size.inspect}" unless size.is_a?(Integer) && size >= 0
|
|
54
|
+
return self if size.zero?
|
|
55
|
+
segment = segment.dup.freeze if segment.is_a?(String) && !segment.frozen?
|
|
56
|
+
segment.freeze if segment.is_a?(Sequence)
|
|
57
|
+
weak = etag.nil? || etag.to_s.start_with?("W/")
|
|
58
|
+
opaque_etag = etag.to_s.delete_prefix("W/").delete_prefix('"').delete_suffix('"')
|
|
59
|
+
@intervals << Interval.new(segment, size, @size, opaque_etag, weak)
|
|
60
|
+
@size += size
|
|
61
|
+
@etag = nil
|
|
62
|
+
self
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Adds a String (sized in bytes and validated by its contents), a Sequence, an object responding
|
|
66
|
+
# to `bytesize`, or an IO-ish object (responding to `read` and `size`, like File, StringIO or Tempfile).
|
|
67
|
+
# Anything else needs an explicit size, so use #add_segment for it.
|
|
68
|
+
#
|
|
69
|
+
# @rbs segment: untyped
|
|
70
|
+
# @rbs return: self
|
|
71
|
+
def <<(segment)
|
|
72
|
+
case segment
|
|
73
|
+
when String
|
|
74
|
+
add_segment(segment, size: segment.bytesize, etag: Digest::SHA1.hexdigest(segment))
|
|
75
|
+
when Sequence
|
|
76
|
+
add_segment(segment, size: segment.size, etag: segment.strong_etag? ? segment.etag : nil)
|
|
77
|
+
else
|
|
78
|
+
# A bare `size` is an element count for an Array or a Hash, so only trust it for IO-ish objects
|
|
79
|
+
size = if segment.respond_to?(:bytesize)
|
|
80
|
+
segment.bytesize
|
|
81
|
+
elsif segment.respond_to?(:read) && segment.respond_to?(:size)
|
|
82
|
+
segment.size
|
|
83
|
+
else
|
|
84
|
+
raise ArgumentError, "Can not tell the size in bytes of a #{segment.class}, use add_segment with an explicit size:"
|
|
85
|
+
end
|
|
86
|
+
add_segment(segment, size: size)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# A weak ETag stops If-Range from matching, so a client won't splice ranges taken from
|
|
91
|
+
# different versions of a segment whose content has changed but whose size has not
|
|
92
|
+
#
|
|
93
|
+
# Returns a quoted ETag derived from the ETags of all the segments, weak (`W/"..."`)
|
|
94
|
+
# if any segment was added without an `etag:`
|
|
95
|
+
#
|
|
96
|
+
# @rbs return: String
|
|
97
|
+
def etag
|
|
98
|
+
@etag ||= begin
|
|
99
|
+
# The size goes in too, since the same validator with a different size is a different representation
|
|
100
|
+
digest = '"%s"' % Digest::SHA1.hexdigest(@intervals.map { |iv| "#{iv.size}:#{iv.etag}" }.join("\0"))
|
|
101
|
+
strong_etag? ? digest : "W/#{digest}"
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Freezes the Sequence, so that no more segments can be added
|
|
106
|
+
#
|
|
107
|
+
# @rbs return: self
|
|
108
|
+
def freeze
|
|
109
|
+
etag # memoize while we still can
|
|
110
|
+
@intervals.freeze
|
|
111
|
+
super
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Tells whether every segment has an ETag of its own
|
|
115
|
+
#
|
|
116
|
+
# @rbs return: bool
|
|
117
|
+
def strong_etag?
|
|
118
|
+
@intervals.none?(&:weak)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Yields every segment the range touches, along with the inclusive range of bytes within that segment.
|
|
122
|
+
# Nested Sequences are descended into, so only the leaf segments get yielded.
|
|
123
|
+
#
|
|
124
|
+
# The range must have both ends given, exclusive ranges are fine. The yielded ranges are always inclusive.
|
|
125
|
+
# Returns an Enumerator when called without a block.
|
|
126
|
+
#
|
|
127
|
+
# @rbs (Range[Integer] range) { (untyped segment, Range[Integer] range_in_segment) -> void } -> void
|
|
128
|
+
# | (Range[Integer] range) -> Enumerator[[untyped, Range[Integer]], void]
|
|
129
|
+
def each_in_range(range, &blk)
|
|
130
|
+
return enum_for(:each_in_range, range) unless blk
|
|
131
|
+
raise ArgumentError, "The range must have both a beginning and an end, but was #{range.inspect}" if range.begin.nil? || range.end.nil?
|
|
132
|
+
first = range.begin
|
|
133
|
+
last = range.exclude_end? ? range.end - 1 : range.end
|
|
134
|
+
raise ArgumentError, "The range must not be reversed, but was #{range.inspect}" if last < first - 1 # 3...3 is just empty
|
|
135
|
+
return if last < first
|
|
136
|
+
|
|
137
|
+
i = @intervals.bsearch_index { |iv| iv.offset + iv.size > first }
|
|
138
|
+
return unless i
|
|
139
|
+
|
|
140
|
+
while (iv = @intervals[i]) && iv.offset <= last
|
|
141
|
+
from = [first, iv.offset].max - iv.offset
|
|
142
|
+
to = [last, iv.offset + iv.size - 1].min - iv.offset
|
|
143
|
+
if iv.segment.is_a?(Sequence)
|
|
144
|
+
iv.segment.each_in_range(from..to, &blk)
|
|
145
|
+
else
|
|
146
|
+
yield(iv.segment, from..to)
|
|
147
|
+
end
|
|
148
|
+
i += 1
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|