serve_byte_range 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 699a0639bd7b8667bba8cb9b53ff26bfd55aaabfec562cfed332979aef0a3702
4
- data.tar.gz: 07e18c40f407e2abd4c46cc2e808bcc42bafda180fc18bda1078d3d6e5c1e394
3
+ metadata.gz: b89324c74c04f679e18acdf0184acb3f82b7b65e96b27bae9216b82ace69840d
4
+ data.tar.gz: e302baff4f9c8cb95569fc9cca6b468526e821ace7a82ab16c4574c3dd75bb33
5
5
  SHA512:
6
- metadata.gz: 14b9ec669046f734e515d0211ee56342bd372ff7ac9d5e9b974b01835340cdb3f58a58caba4e23e82e8de9cb0ddb27a71dc36442f4e07862087bc88a51ee8510
7
- data.tar.gz: cf3260ffd5b00d9c82d6dfc4eccf802506c8247c595ac26e479641c3bfb526ef3baea8434d4f0672d14dafc323456b3d9157950cc9e4005d2eac1840ac487940
6
+ metadata.gz: 4ade7e302976c27c11d19d5541e4e0b2a524011f60091fa6ad6b7439882cd3b7ece4515c2791febe7f47de7a9a1bd1d591161a2e09ec40290489cdc4e0f4e433
7
+ data.tar.gz: 285b227ea76fff572f399a2492d2e7ef02d89a178a7c793a78568627bb16b408fccddead4417d6cd1fbc70a37f0c1eedbbe23c23a6b821e32ba70e9424123bf0
@@ -7,24 +7,44 @@ env:
7
7
  BUNDLE_PATH: vendor/bundle
8
8
 
9
9
  jobs:
10
- test_and_lint_minimum_ruby:
11
- name: Tests/Lint (2.7)
10
+ test:
11
+ name: Tests (Ruby ${{ matrix.ruby }}, Rack ${{ matrix.rack }})
12
12
  runs-on: ubuntu-22.04
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ ruby: ["2.7", "3.4"]
17
+ rack: ["2", "3"]
18
+ env:
19
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/rack_${{ matrix.rack }}.gemfile
13
20
  steps:
14
21
  - name: Checkout
15
22
  uses: actions/checkout@v4
16
23
  - name: Setup Ruby
17
24
  uses: ruby/setup-ruby@v1
18
25
  with:
19
- ruby-version: 2.7
26
+ ruby-version: ${{ matrix.ruby }}
20
27
  bundler-cache: true
21
28
  - name: "Tests"
22
29
  run: bundle exec rake test --backtrace
30
+ lint:
31
+ name: Lint (2.7)
32
+ runs-on: ubuntu-22.04
33
+ steps:
34
+ - name: Checkout
35
+ uses: actions/checkout@v4
36
+ - name: Setup Ruby
37
+ uses: ruby/setup-ruby@v1
38
+ with:
39
+ ruby-version: 2.7
40
+ bundler-cache: true
23
41
  - name: "Lint"
24
42
  run: bundle exec rake standard
25
- test_recent_ruby:
26
- name: Tests (3.4)
43
+ typecheck:
44
+ name: Typecheck (3.4)
27
45
  runs-on: ubuntu-22.04
46
+ env:
47
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/typecheck.gemfile
28
48
  steps:
29
49
  - name: Checkout
30
50
  uses: actions/checkout@v4
@@ -33,5 +53,9 @@ jobs:
33
53
  with:
34
54
  ruby-version: 3.4
35
55
  bundler-cache: true
36
- - name: "Tests"
37
- run: bundle exec rake test --backtrace
56
+ - name: "Steep"
57
+ run: bundle exec rake steep
58
+ - name: "Signatures in sig/ are up to date"
59
+ run: |
60
+ git add --intent-to-add sig
61
+ git diff --exit-code sig || (echo "Run 'rake rbs' and commit the changes in sig/" && exit 1)
data/.gitignore CHANGED
@@ -5,9 +5,8 @@ coverage.data
5
5
  # rdoc generated
6
6
  rdoc
7
7
 
8
- # yard generated
8
+ # rdoc generated
9
9
  doc
10
- .yardoc
11
10
 
12
11
  # Rubocop
13
12
  rubocop.html
@@ -56,3 +55,5 @@ tmp
56
55
  rubocop.html
57
56
 
58
57
  pkg/*.gem
58
+
59
+ gemfiles/*.lock
data/.standard.yml CHANGED
@@ -1 +1,5 @@
1
1
  ruby_version: 2.7
2
+ ignore:
3
+ # `#: Type` is rbs-inline/Steep annotation syntax, which the RuboCop pinned by our Standard predates
4
+ - "lib/**/*.rb":
5
+ - Layout/LeadingCommentSpace
data/README.md CHANGED
@@ -28,3 +28,61 @@ You can also retrieve the `range_in_file` from an external resource, you can als
28
28
  * Will be correctly pre-sized with `Content-Length`
29
29
  * Will not be generated and buffered eagerly, but will get generated as you serve the content out
30
30
 
31
+ Every string you `write` gets copied before it is passed on to the webserver, since `IO.copy_stream` and `IO#readpartial` with a buffer reuse the same `String` for every chunk, and middleware like `Rack::ETag` holds on to the chunks. Strings which are frozen and already in the `BINARY` encoding are passed on as-is, as nobody can change them afterwards.
32
+
33
+
34
+ ## Serving a resource made of several segments
35
+
36
+ If your response is glued together from several pieces - byte ranges of objects in S3, literal strings and so on - use a `ServeByteRange::Sequence`. A segment can be any object, since you give its size explicitly. For every requested range you get the segments it touches, along with the range of bytes inside each segment:
37
+
38
+ ```ruby
39
+ header_file = File.open("header.bin", "rb")
40
+
41
+ seq = ServeByteRange::Sequence.new
42
+ seq.add_segment(header_file, size: header_file.size, etag: "#{header_file.mtime.to_i}-#{header_file.size}")
43
+ seq.add_segment(s3_object, size: s3_object.content_length, etag: s3_object.etag)
44
+ seq << "--- redacted ---\n" # Strings are sized in bytes, and their contents go into the ETag
45
+ seq.add_segment(other_s3_object, size: other_s3_object.content_length, etag: other_s3_object.etag)
46
+
47
+ status, headers, body = ServeByteRange.serve_sequence(env, seq) do |segment, range_in_segment, io|
48
+ case segment
49
+ when String
50
+ io.write(segment.byteslice(range_in_segment))
51
+ when File
52
+ # Copies range_in_segment.size bytes starting at range_in_segment.begin, without moving the file position
53
+ IO.copy_stream(segment, io, range_in_segment.size, range_in_segment.begin)
54
+ when Aws::S3::Object
55
+ # Passing a block to `get` streams the body in chunks instead of buffering it in memory
56
+ segment.get(range: "bytes=%d-%d" % [range_in_segment.begin, range_in_segment.end]) do |chunk|
57
+ io.write(chunk)
58
+ end
59
+ end
60
+ end
61
+
62
+ # The webserver calls `close` on the body once the response is sent, or if the client goes away
63
+ [status, headers, Rack::BodyProxy.new(body) { header_file.close }]
64
+ ```
65
+
66
+ The body is generated lazily while the webserver sends the response, so the `File` has to stay open until then. Closing it right after `serve_sequence` returns would break the response, and not closing it at all leaks a file descriptor per request - hence the `Rack::BodyProxy`.
67
+
68
+ The `ETag` of the response is derived from the ETags of the segments. If you do not give an `etag:` for a segment, a change which keeps its size the same will not be noticed, so the `ETag` of the `Sequence` becomes weak (`W/"..."`). Conditional GETs keep working with a weak `ETag`, but `If-Range` won't match it, so clients resuming a download get the whole response again instead of stitching together bytes from two different versions. Sequences can be nested. Segments can't change once added: Strings get copied and frozen, and a `Sequence` gets frozen when it goes into another one. If a segment writes more or fewer bytes than its range requires, you get an exception saying which segment it was.
69
+
70
+ ## Development
71
+
72
+ Range parsing is delegated to Rack, and its behavior differs between Rack versions. To run the tests against a specific major version of Rack:
73
+
74
+ ```shell
75
+ BUNDLE_GEMFILE=gemfiles/rack_2.gemfile bundle install
76
+ BUNDLE_GEMFILE=gemfiles/rack_2.gemfile bundle exec rake test
77
+ ```
78
+
79
+ CI runs the tests against both `gemfiles/rack_2.gemfile` and `gemfiles/rack_3.gemfile`.
80
+
81
+ Types are written inline as [rbs-inline](https://github.com/soutaro/rbs-inline) annotations, and the RBS signatures in `sig/` are generated from them - so after changing a signature, regenerate them and typecheck with Steep. rbs-inline and Steep need a recent Ruby, so they live in a Gemfile of their own:
82
+
83
+ ```shell
84
+ BUNDLE_GEMFILE=gemfiles/typecheck.gemfile bundle install
85
+ bundle exec rake steep # runs `rake rbs` first
86
+ ```
87
+
88
+ API docs are RDoc with Markdown markup, `bundle exec rake rdoc` writes them into `doc/`.
data/Rakefile CHANGED
@@ -3,6 +3,7 @@
3
3
  require "bundler/gem_tasks"
4
4
  require "rake/testtask"
5
5
  require "standard/rake"
6
+ require "rdoc/task"
6
7
 
7
8
  Rake::TestTask.new(:test) do |t|
8
9
  t.libs << "test"
@@ -15,12 +16,24 @@ task :format do
15
16
  `bundle exec magic_frozen_string_literal .`
16
17
  end
17
18
 
18
- task :generate_typedefs do
19
- `bundle exec sord rbi/serve_byte_range.rbi`
19
+ RDoc::Task.new(:rdoc) do |rd|
20
+ rd.rdoc_dir = "doc"
21
+ rd.main = "README.md"
22
+ rd.markup = "markdown"
23
+ rd.rdoc_files.include("README.md", "lib/**/*.rb")
20
24
  end
21
25
 
22
- # When building the gem, generate typedefs beforehand,
23
- # so that they get included
24
- Rake::Task["build"].enhance(["generate_typedefs"])
26
+ # rbs-inline and steep need Ruby 3.1+, so they live in a Gemfile of their own
27
+ TYPECHECK_ENV = {"BUNDLE_GEMFILE" => File.expand_path("gemfiles/typecheck.gemfile", __dir__)}
25
28
 
26
- task default: [:test, :standard, :generate_typedefs]
29
+ desc "Generate RBS signatures in sig/ from the inline annotations in lib/"
30
+ task :rbs do
31
+ sh TYPECHECK_ENV, "bundle exec rbs-inline --output=sig lib"
32
+ end
33
+
34
+ desc "Typecheck with Steep"
35
+ task steep: :rbs do
36
+ sh TYPECHECK_ENV, "bundle exec steep check"
37
+ end
38
+
39
+ task default: [:test, :standard]
data/Steepfile ADDED
@@ -0,0 +1,10 @@
1
+ D = Steep::Diagnostic
2
+
3
+ target :lib do
4
+ signature "sig"
5
+ # Kept out of sig/ so that it does not ship with the gem and clash with Rack signatures apps may have
6
+ signature "sig_shims"
7
+ check "lib"
8
+ library "digest"
9
+ configure_code_diagnostics(D::Ruby.default)
10
+ end
@@ -0,0 +1,5 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec path: ".."
4
+
5
+ gem "rack", "~> 2.0"
@@ -0,0 +1,5 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec path: ".."
4
+
5
+ gem "rack", "~> 3.0"
@@ -0,0 +1,7 @@
1
+ source "https://rubygems.org"
2
+
3
+ gemspec path: ".."
4
+
5
+ # rbs-inline and steep need Ruby 3.1+, so they can't go into the gemspec while we test on 2.7
6
+ gem "rbs-inline", require: false
7
+ gem "steep", require: false
@@ -0,0 +1,152 @@
1
+ # frozen_string_literal: true
2
+
3
+ # rbs_inline: enabled
4
+
5
+ require "digest"
6
+
7
+ module ServeByteRange
8
+ # A resource composed of several segments glued together - for example, byte ranges of objects
9
+ # in remote storage interleaved with literal strings. A segment can be any object, since its
10
+ # size is given explicitly. A Sequence can also be a segment of another Sequence.
11
+ #
12
+ # seq = ServeByteRange::Sequence.new
13
+ # seq.add_segment(s3_object, size: 1024, etag: s3_object.etag)
14
+ # seq << "Literal text"
15
+ # ServeByteRange.serve_sequence(env, seq) do |segment, range_in_segment, io|
16
+ # io.write(segment.is_a?(String) ? segment.byteslice(range_in_segment) : fetch_range(segment, range_in_segment))
17
+ # end
18
+ class Sequence
19
+ Interval = Struct.new(
20
+ :segment, #: untyped
21
+ :size, #: Integer
22
+ :offset, #: Integer
23
+ :etag, #: String
24
+ :weak #: bool
25
+ )
26
+ private_constant :Interval
27
+
28
+ # @rbs @intervals: Array[Interval]
29
+ # @rbs @etag: String?
30
+
31
+ # The total size of all segments in bytes
32
+ attr_reader :size #: Integer
33
+
34
+ # @rbs *segments: untyped -- segments to add with `<<`
35
+ def initialize(*segments)
36
+ @intervals = []
37
+ @size = 0
38
+ segments.each { |segment| self << segment }
39
+ end
40
+
41
+ # Adds a segment of a given size. Segments of zero size are skipped. The size and ETag are taken
42
+ # as they are now, so a String gets copied and frozen, and a Sequence gets frozen when added.
43
+ #
44
+ # The `etag` goes into the ETag of the Sequence. Without it only the size is known, which won't catch
45
+ # content changes that keep the size the same - so the ETag of the Sequence becomes weak. It can be
46
+ # quoted (as S3 returns it) or not, and a weak one (`W/"..."`) makes the ETag of the Sequence weak too.
47
+ #
48
+ # @rbs segment: untyped -- any object, it gets yielded back when serving
49
+ # @rbs size: Integer -- the size of the segment in bytes
50
+ # @rbs etag: untyped -- a validator for the segment
51
+ # @rbs return: self
52
+ def add_segment(segment, size:, etag: nil)
53
+ raise ArgumentError, "Segment size must be an Integer >= 0, but was #{size.inspect}" unless size.is_a?(Integer) && size >= 0
54
+ return self if size.zero?
55
+ segment = segment.dup.freeze if segment.is_a?(String) && !segment.frozen?
56
+ segment.freeze if segment.is_a?(Sequence)
57
+ weak = etag.nil? || etag.to_s.start_with?("W/")
58
+ opaque_etag = etag.to_s.delete_prefix("W/").delete_prefix('"').delete_suffix('"')
59
+ @intervals << Interval.new(segment, size, @size, opaque_etag, weak)
60
+ @size += size
61
+ @etag = nil
62
+ self
63
+ end
64
+
65
+ # Adds a String (sized in bytes and validated by its contents), a Sequence, an object responding
66
+ # to `bytesize`, or an IO-ish object (responding to `read` and `size`, like File, StringIO or Tempfile).
67
+ # Anything else needs an explicit size, so use #add_segment for it.
68
+ #
69
+ # @rbs segment: untyped
70
+ # @rbs return: self
71
+ def <<(segment)
72
+ case segment
73
+ when String
74
+ add_segment(segment, size: segment.bytesize, etag: Digest::SHA1.hexdigest(segment))
75
+ when Sequence
76
+ add_segment(segment, size: segment.size, etag: segment.strong_etag? ? segment.etag : nil)
77
+ else
78
+ # A bare `size` is an element count for an Array or a Hash, so only trust it for IO-ish objects
79
+ size = if segment.respond_to?(:bytesize)
80
+ segment.bytesize
81
+ elsif segment.respond_to?(:read) && segment.respond_to?(:size)
82
+ segment.size
83
+ else
84
+ raise ArgumentError, "Can not tell the size in bytes of a #{segment.class}, use add_segment with an explicit size:"
85
+ end
86
+ add_segment(segment, size: size)
87
+ end
88
+ end
89
+
90
+ # A weak ETag stops If-Range from matching, so a client won't splice ranges taken from
91
+ # different versions of a segment whose content has changed but whose size has not
92
+ #
93
+ # Returns a quoted ETag derived from the ETags of all the segments, weak (`W/"..."`)
94
+ # if any segment was added without an `etag:`
95
+ #
96
+ # @rbs return: String
97
+ def etag
98
+ @etag ||= begin
99
+ # The size goes in too, since the same validator with a different size is a different representation
100
+ digest = '"%s"' % Digest::SHA1.hexdigest(@intervals.map { |iv| "#{iv.size}:#{iv.etag}" }.join("\0"))
101
+ strong_etag? ? digest : "W/#{digest}"
102
+ end
103
+ end
104
+
105
+ # Freezes the Sequence, so that no more segments can be added
106
+ #
107
+ # @rbs return: self
108
+ def freeze
109
+ etag # memoize while we still can
110
+ @intervals.freeze
111
+ super
112
+ end
113
+
114
+ # Tells whether every segment has an ETag of its own
115
+ #
116
+ # @rbs return: bool
117
+ def strong_etag?
118
+ @intervals.none?(&:weak)
119
+ end
120
+
121
+ # Yields every segment the range touches, along with the inclusive range of bytes within that segment.
122
+ # Nested Sequences are descended into, so only the leaf segments get yielded.
123
+ #
124
+ # The range must have both ends given, exclusive ranges are fine. The yielded ranges are always inclusive.
125
+ # Returns an Enumerator when called without a block.
126
+ #
127
+ # @rbs (Range[Integer] range) { (untyped segment, Range[Integer] range_in_segment) -> void } -> void
128
+ # | (Range[Integer] range) -> Enumerator[[untyped, Range[Integer]], void]
129
+ def each_in_range(range, &blk)
130
+ return enum_for(:each_in_range, range) unless blk
131
+ raise ArgumentError, "The range must have both a beginning and an end, but was #{range.inspect}" if range.begin.nil? || range.end.nil?
132
+ first = range.begin
133
+ last = range.exclude_end? ? range.end - 1 : range.end
134
+ raise ArgumentError, "The range must not be reversed, but was #{range.inspect}" if last < first - 1 # 3...3 is just empty
135
+ return if last < first
136
+
137
+ i = @intervals.bsearch_index { |iv| iv.offset + iv.size > first }
138
+ return unless i
139
+
140
+ while (iv = @intervals[i]) && iv.offset <= last
141
+ from = [first, iv.offset].max - iv.offset
142
+ to = [last, iv.offset + iv.size - 1].min - iv.offset
143
+ if iv.segment.is_a?(Sequence)
144
+ iv.segment.each_in_range(from..to, &blk)
145
+ else
146
+ yield(iv.segment, from..to)
147
+ end
148
+ i += 1
149
+ end
150
+ end
151
+ end
152
+ end
@@ -1,3 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module ServeByteRange
2
- VERSION = "1.0.0"
4
+ VERSION = "1.1.0"
3
5
  end
6
+
7
+ # rbs_inline: enabled