interval_response 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2ead31e0eeba26a2752056561f041073652cc485aa7d34f9e939792ec990d0c1
+  data.tar.gz: 1bc76de0fe94c1943be22fc9674007021778b80f494f2aeacfc31f81db08e0e9
+SHA512:
+  metadata.gz: d217a6cc297e54d1955b76c9919d1ee7259f3a9d157d13a473a2410656824316906e3805b37d7fd14ae9f1c3f7681900740368e619e56bb023d071d2274ed4d2
+  data.tar.gz: 21e8766bf398ef4a00f7e81b34214da43f13bbc20b1ed8b2e89a83c305f0a5ce43764cd7024626c7367e1c40a70d4c73334e23b6d811c67f14a898e736f9cb28

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+Gemfile.lock
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,3 @@
+inherit_gem:
+  wetransfer_style: ruby/default.yml
+

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.17.3

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in interval_response.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Julik Tarkhanov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,79 @@
+# IntervalResponse
+
+is a little piece of machinery which allows your Rack/Rails application to correctly
+serve HTTP `Range:` responses. Features:
+
+* Strong ETags depending on response composition
+* Correct response codes/headers/offsets
+* `multipart/byte-range` responses
+* Segments comprising the body do not have to be materialized into buffers or strings prior to serving
+* Responds to both `GET` and `HEAD`, to the latter without body
+* Is [measurometer](https://github.com/WeTransfer/measurometer)-instrumented
+
+## Usage
+
+Imagine you have a number of long Strings you want to serve concatenated as a single HTTP resource.
+Wrap them in an `IntervalResponse` and return it to Rack:
+
+```
+verses_app = ->(env) {
+  all_verses = ImportantVerse.all.map(&:verse_text)
+  interval_sequence = IntervalResponse::Sequence.new(*all_verses)
+  response = IntervalResponse.new(interval_sequence, env['HTTP_RANGE'], env['HTTP_IF_RANGE'])
+  response.to_rack_response_triplet
+}
+```
+
+Or imagine you want to serve out a few very large log files, concatenated together
+
+```
+  log_paths = Dir.glob('/tmp/logs/kafkadoop.*.log.gz').sort
+  # Wrap them with "lazy file" proxies so that the files
+  # do not have to stay open during the entire response output
+  lazy_files = log_paths.map { |path| IntervalResponse::LazyFile.new(path) }
+  interval_sequence = IntervalResponse::Sequence.new(*lazy_files)
+  response = IntervalResponse.new(interval_sequence, env['HTTP_RANGE'], env['HTTP_IF_RANGE'])
+  response.to_rack_response_triplet(headers: {'X-Server' => 'teapot'})
+```
+
+Note that the headers `IntervalResponse` generates are _very_ specific and will override your
+headers. The following headers will be overridden (as they must all be correct for the serving
+to work):
+
+```
+Accept-Ranges
+Content-Length
+Content-Type
+Content-Range
+ETag
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'interval_response'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install interval_response
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/julik/interval_response.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:rubocop)
+task default: [:spec, :rubocop]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "interval_response"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/interval_response.gemspec

@@ -0,0 +1,46 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "interval_response/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "interval_response"
+  spec.version       = IntervalResponse::VERSION
+  spec.authors       = ["Julik Tarkhanov"]
+  spec.email         = ["me@julik.nl"]
+
+  spec.summary       = %q{Assemble HTTP responses from spliced sequences of payloads}
+  spec.description   = %q{Assemble HTTP responses from spliced sequences of payloads}
+  spec.homepage      = "https://github.com/WeTransfer/interval_response"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata["homepage_uri"] = spec.homepage
+    spec.metadata["source_code_uri"] = "https://github.com/WeTransfer/interval_response"
+    spec.metadata["changelog_uri"] = "https://github.com/WeTransfer/interval_response"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0")
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "rack"
+  spec.add_runtime_dependency "measurometer"
+
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 12"
+  spec.add_development_dependency "rspec", "~> 3"
+  spec.add_development_dependency "complexity_assert"
+  spec.add_development_dependency "rack-test"
+  spec.add_development_dependency "wetransfer_style"
+end

data/lib/interval_response.rb

@@ -0,0 +1,62 @@
+require 'measurometer'
+require 'rack'
+
+module IntervalResponse
+  class Error < StandardError; end
+
+  require_relative "interval_response/version"
+  require_relative "interval_response/to_rack_response_triplet"
+  require_relative "interval_response/sequence"
+  require_relative "interval_response/empty"
+  require_relative "interval_response/single"
+  require_relative "interval_response/invalid"
+  require_relative "interval_response/multi"
+  require_relative "interval_response/full"
+  require_relative "interval_response/lazy_file"
+
+  ENTIRE_RESOURCE_RANGE = 'bytes=0-'
+
+  def self.new(interval_map, http_range_header_value_or_nil, http_if_range_header_or_nil)
+    # If the 'If-Range' header is provided but does not match, discard the Range header. It means
+    # that the client is requesting a certain representation of the resource and wants a range
+    # _within_ that representation, but the representation has since changed and the offsets
+    # no longer make sense. In that case we are supposed to answer with a 200 and the full
+    # monty.
+    return new(interval_map, ENTIRE_RESOURCE_RANGE, nil) if http_if_range_header_or_nil && http_if_range_header_or_nil != interval_map.etag
+
+    if http_if_range_header_or_nil
+      Measurometer.increment_counter('interval_response.if_range_provided')
+    elsif http_range_header_value_or_nil
+      Measurometer.increment_counter('interval_response.if_range_not_provided')
+    end
+
+    prepare_response(interval_map, http_range_header_value_or_nil, http_if_range_header_or_nil).tap do |res|
+      response_type_name_for_metric = res.class.to_s.split('::').last.downcase # Some::Module::Empty => empty
+      Measurometer.increment_counter('interval_response.resp_%s' % response_type_name_for_metric)
+    end
+  end
+
+  def self.prepare_response(interval_map, http_range_header_value_or_nil, _http_if_range_header_or_nil)
+    # Case 1 - response of 0 bytes (empty resource).
+    # We don't even have to parse the Range header for this since
+    # the response will be the same, always.
+    return Empty.new(interval_map) if interval_map.empty?
+
+    # Parse the HTTP Range: header
+    range_request_header = http_range_header_value_or_nil || ENTIRE_RESOURCE_RANGE
+    http_ranges = Rack::Utils.get_byte_ranges(range_request_header, interval_map.size)
+
+    # Case 2 - Client did send us a Range header, but Rack discarded
+    # it because it is invalid and cannot be satisfied
+    return Invalid.new(interval_map) if http_range_header_value_or_nil && http_ranges.empty?
+
+    # Case 3 - entire resource
+    return Full.new(interval_map) if http_ranges.length == 1 && http_ranges.first == (0..(interval_map.size - 1))
+
+    # Case 4 - one content range
+    return Single.new(interval_map, http_ranges) if http_ranges.length == 1
+
+    # Case 5 - MIME multipart with multiple content ranges
+    Multi.new(interval_map, http_ranges)
+  end
+end

data/lib/interval_response/empty.rb

@@ -0,0 +1,29 @@
+# Serves out a response that is of size 0
+class IntervalResponse::Empty
+  include IntervalResponse::ToRackResponseTriplet
+
+  def initialize(interval_map)
+    @interval_map = interval_map
+  end
+
+  def each
+    # No-op
+  end
+
+  def status_code
+    200
+  end
+
+  def content_length
+    0
+  end
+
+  def headers
+    {
+      'Accept-Ranges' => 'bytes',
+      'Content-Length' => '0',
+      'Content-Type' => 'binary/octet-stream',
+      'ETag' => @interval_map.etag,
+    }
+  end
+end

data/lib/interval_response/full.rb

@@ -0,0 +1,33 @@
+# Serves out a response that contains the entire resource
+class IntervalResponse::Full
+  include IntervalResponse::ToRackResponseTriplet
+
+  def initialize(interval_map, *)
+    @interval_map = interval_map
+  end
+
+  def each
+    # serve the part of the interval map
+    full_range = 0..(@interval_map.size - 1)
+    @interval_map.each_in_range(full_range) do |segment, range_in_segment|
+      yield(segment, range_in_segment)
+    end
+  end
+
+  def status_code
+    200
+  end
+
+  def content_length
+    @interval_map.size
+  end
+
+  def headers
+    {
+      'Accept-Ranges' => 'bytes',
+      'Content-Length' => @interval_map.size.to_s,
+      'Content-Type' => 'binary/octet-stream',
+      'ETag' => @interval_map.etag,
+    }
+  end
+end

data/lib/interval_response/invalid.rb

@@ -0,0 +1,33 @@
+# Serves out a response that is of size 0
+class IntervalResponse::Invalid
+  include IntervalResponse::ToRackResponseTriplet
+
+  ERROR_JSON = '{"message": "Ranges cannot be satisfied"}'
+
+  def initialize(segment_map)
+    @interval_map = segment_map
+  end
+
+  def each
+    full_segment_range = (0..(ERROR_JSON.bytesize - 1))
+    yield(ERROR_JSON, full_segment_range)
+  end
+
+  def status_code
+    416
+  end
+
+  def content_length
+    ERROR_JSON.bytesize
+  end
+
+  def headers
+    {
+      'Accept-Ranges' => 'bytes',
+      'Content-Length' => ERROR_JSON.bytesize.to_s,
+      'Content-Type' => 'application/json',
+      'Content-Range' => "bytes */#{@interval_map.size}",
+      'ETag' => @interval_map.etag,
+    }
+  end
+end

data/lib/interval_response/lazy_file.rb

@@ -0,0 +1,19 @@
+# Used so that if a sequence of files
+# gets served out, the files should not be kept open
+# during the entire response output - as this might
+# exhaust the file descriptor table
+class IntervalResponse::LazyFile
+  def initialize(filesystem_path)
+    @fs_path = filesystem_path
+  end
+
+  def size
+    File.size(@fs_path)
+  end
+
+  def with
+    File.open(@fs_path, 'rb') do |file_handle|
+      yield file_handle
+    end
+  end
+end

data/lib/interval_response/multi.rb

@@ -0,0 +1,72 @@
+require 'securerandom'
+
+class IntervalResponse::Multi
+  include IntervalResponse::ToRackResponseTriplet
+
+  ALPHABET = ('0'..'9').to_a + ('a'..'z').to_a + ('A'..'Z').to_a
+
+  def initialize(interval_map, http_ranges)
+    @interval_map = interval_map
+    @http_ranges = http_ranges
+    # RFC1521 says that a boundary "must be no longer than 70 characters,
+    # not counting the two leading hyphens".
+    # Modulo-based random is biased but it doesn't matter much for us (we do not need to
+    # be extremely secure here)
+    @boundary = SecureRandom.bytes(24).unpack("C*").map { |b| ALPHABET[b % ALPHABET.length] }.join
+  end
+
+  def each
+    # serve the part of the interval map
+    @http_ranges.each_with_index do |http_range, range_i|
+      part_header = part_header(range_i, http_range)
+      entire_header_range = 0..(part_header.bytesize - 1)
+      yield(part_header, entire_header_range)
+      @interval_map.each_in_range(http_range) do |segment, range_in_segment|
+        yield(segment, range_in_segment)
+      end
+    end
+  end
+
+  def status_code
+    206
+  end
+
+  def content_length
+    # The Content-Length of a multipart response includes the length
+    # of all the ranges of the resource, but also the lengths of the
+    # multipart part headers - which we need to precompute. To do it
+    # we need to run through all of our ranges and output some strings,
+    # and if a lot of ranges are involved this can get expensive. So
+    # memoize the envelope size (it never changes between calls)
+    @envelope_size ||= compute_envelope_size
+  end
+
+  def headers
+    {
+      'Accept-Ranges' => 'bytes',
+      'Content-Length' => content_length.to_s,
+      'Content-Type' => "multipart/byte-ranges; boundary=#{@boundary}",
+      'ETag' => @interval_map.etag,
+    }
+  end
+
+  private
+
+  def compute_envelope_size
+    @http_ranges.each_with_index.inject(0) do |size_sum, (http_range, part_index)|
+      header_bytes = part_header(part_index, http_range)
+      range_size = http_range.end - http_range.begin + 1
+      size_sum + header_bytes.bytesize + range_size
+    end
+  end
+
+  def part_header(part_index, http_r)
+    [
+      part_index > 0 ? "\r\n" : "", # Parts follwing the first have to be delimited "at the top"
+      "--%s\r\n" % @boundary,
+      "Content-Type: binary/octet-stream\r\n",
+      "Content-Range: bytes %d-%d/%d\r\n" % [http_r.begin, http_r.end, @interval_map.size],
+      "\r\n",
+    ].join
+  end
+end

data/lib/interval_response/sequence.rb

@@ -0,0 +1,127 @@
+require 'digest'
+
+# An interval sequence represents a linear sequence of non-overlapping,
+# joined intervals. For example, an HTTP response which consists of
+# multiple edge included segments, or a timeline with clips joined together.
+# Every interval contains a *segment* - an arbitrary object which responds to
+# `#size` at time of adding to the IntervalSequence.
+class IntervalResponse::Sequence
+  MULTIPART_GENRATOR_FINGERPRINT = 'boo'
+  Interval = Struct.new(:segment, :size, :offset, :position)
+
+  attr_reader :size
+
+  def initialize(*segments)
+    @intervals = []
+    @size = 0
+    segments.each { |s| self << s }
+  end
+
+  def <<(segment)
+    return self if segment.size == 0
+    segment_size_or_bytesize = segment.respond_to?(:bytesize) ? segment.bytesize : segment.size
+    @intervals << Interval.new(segment, segment_size_or_bytesize, @size, @intervals.length)
+    @size += segment.size
+    self
+  end
+
+  def each_in_range(from_range_in_resource)
+    # Skip empty ranges
+    requested_range_size = (from_range_in_resource.end - from_range_in_resource.begin) + 1
+    return if requested_range_size < 1
+
+    # ...and if the range misses our intervals completely
+    included_intervals = intervals_within_range(from_range_in_resource)
+
+    # And normal case - walk through included intervals
+    included_intervals.each do |interval|
+      int_start = interval.offset
+      int_end = interval.offset + interval.size - 1
+      req_start = from_range_in_resource.begin
+      req_end = from_range_in_resource.end
+      range_within_interval = (max(int_start, req_start) - int_start)..(min(int_end, req_end) - int_start)
+
+      # Allow Sequences to be composed together
+      if interval.segment.respond_to?(:each_in_range)
+        interval.segment.each_in_range(range_within_interval) do |sub_segment, sub_range|
+          yield(sub_segment, sub_range)
+        end
+      else
+        yield(interval.segment, range_within_interval)
+      end
+    end
+  end
+
+  def empty?
+    @size == 0
+  end
+
+  # For IE resumes to work, a strong ETag must be set in the response, and a strong
+  # comparison must be performed on it.
+  #
+  # ETags have meaning with Range: requests, because when a client requests
+  # a range it will send the ETag back in the If-Range header. That header
+  # tells the server that "I want to have the ranges as emitted by the
+  # response representation that has output this etag". This is done so that
+  # there is a guarantee that the same resource being requested has the same
+  # resource length (off of which the ranges get computed), and the ranges
+  # can be safely combined by the client. In practice this means that the ETag
+  # must contain some "version handle" which stays unchanged as long as the code
+  # responsible for generating the response does not change. In our case the response
+  # can change due to the following things:
+  #
+  # * The lengths of the segments change
+  # * The contents of the segments changes
+  # * Code that outputs the ranges themselves changes, and outputs different offsets of differently-sized resources.
+  #   A resource _can_ be differently sized since the MIME multiplart-byte-range response can have its boundary
+  #   or per-part headers change, which affects the _size_ of the MIME part headers. Even though the boundary is
+  #   not a part of the resource itself, the sizes of the part headers *do* contribute to the envelope size - that
+  #   should stay the same as long as the ETag holds.
+  #
+  # It is important that the returned ETag is a strong ETag (not prefixed with 'W/') and must be
+  # enclosed in double-quotes.
+  #
+  # See for more https://blogs.msdn.microsoft.com/ieinternals/2011/06/03/download-resumption-in-internet-explorer/
+  def etag
+    d = Digest::SHA1.new
+    d << IntervalResponse::VERSION
+    d << Marshal.dump(@intervals.map(&:size))
+    '"%s"' % d.hexdigest
+  end
+
+  private
+
+  def max(a, b)
+    a > b ? a : b
+  end
+
+  def min(a, b)
+    a < b ? a : b
+  end
+
+  def interval_under(offset)
+    @intervals.bsearch do |interval|
+      # bsearch expects a 0 return value for "exact match".
+      # -1 tells it "look to my left" and 1 "look to my right",
+      # which is the output of the <=> operator. If we only needed
+      # to find the exact offset in a sorted list just <=> would be
+      # fine, but since we are looking for offsets within intervals
+      # we will expand the the "match" case with "falls within interval".
+      if offset >= interval.offset && offset < (interval.offset + interval.size)
+        0
+      else
+        offset <=> interval.offset
+      end
+    end
+  end
+
+  def intervals_within_range(http_range)
+    first_touched = interval_under(http_range.begin)
+
+    # The range starts to the right of available range
+    return [] unless first_touched
+
+    last_touched = interval_under(http_range.end) || @intervals.last
+    @intervals[first_touched.position..last_touched.position]
+  end
+end

data/lib/interval_response/single.rb

@@ -0,0 +1,36 @@
+# Serves out a response that consists of one HTTP Range,
+# which is always not the entire resource
+class IntervalResponse::Single
+  include IntervalResponse::ToRackResponseTriplet
+
+  def initialize(interval_map, http_ranges)
+    @interval_map = interval_map
+    @http_range = http_ranges.first
+  end
+
+  # Serve the part of the interval map
+  def each
+    @interval_map.each_in_range(@http_range) do |segment, range_in_segment|
+      yield(segment, range_in_segment)
+    end
+  end
+
+  def status_code
+    206
+  end
+
+  def content_length
+    @http_range.end - @http_range.begin + 1
+  end
+
+  def headers
+    c_range = ('bytes %d-%d/%d' % [@http_range.begin, @http_range.end, @interval_map.size])
+    {
+      'Accept-Ranges' => 'bytes',
+      'Content-Length' => (@http_range.end - @http_range.begin + 1).to_s,
+      'Content-Type' => 'binary/octet-stream',
+      'Content-Range' => c_range,
+      'ETag' => @interval_map.etag,
+    }
+  end
+end

data/lib/interval_response/to_rack_response_triplet.rb

@@ -0,0 +1,58 @@
+module IntervalResponse::ToRackResponseTriplet
+  CHUNK_SIZE = 65 * 1024 # Roughly one TCP kernel buffer
+
+  class RackBodyWrapper
+    def initialize(with_interval_response, chunk_size:)
+      @chunk_size = chunk_size
+      @interval_response = with_interval_response
+    end
+
+    def each
+      buf = String.new(capacity: @chunk_size)
+      @interval_response.each do |segment, range_in_segment|
+        case segment
+        when IntervalResponse::LazyFile
+          segment.with do |_file_handle|
+            with_each_chunk(range_in_segment) do |offset, read_n|
+              segment.seek(offset, IO::SEEK_SET)
+              yield segment.read_nonblock(read_n, buf)
+            end
+          end
+        when String
+          with_each_chunk(range_in_segment) do |offset, read_n|
+            yield segment.slice(offset, read_n)
+          end
+        when IO, Tempfile
+          with_each_chunk(range_in_segment) do |offset, read_n|
+            segment.seek(offset, IO::SEEK_SET)
+            yield segment.read_nonblock(read_n, buf)
+          end
+        else
+          raise TypeError, "RackBodyWrapper only supports IOs or Strings"
+        end
+      end
+    ensure
+      buf.clear
+    end
+
+    def with_each_chunk(range_in_segment)
+      range_size = range_in_segment.end - range_in_segment.begin + 1
+      start_at_offset = range_in_segment.begin
+      n_whole_segments, remainder = range_size.divmod(@chunk_size)
+
+      n_whole_segments.times do |n|
+        unit_offset = start_at_offset + (n * @chunk_size)
+        yield unit_offset, @chunk_size
+      end
+
+      if remainder > 0
+        unit_offset = start_at_offset + (n_whole_segments * @chunk_size)
+        yield unit_offset, remainder
+      end
+    end
+  end
+
+  def to_rack_response_triplet(headers: nil, chunk_size: CHUNK_SIZE)
+    [status_code, headers.to_h.merge(self.headers), RackBodyWrapper.new(self, chunk_size: chunk_size)]
+  end
+end

data/lib/interval_response/version.rb

@@ -0,0 +1,3 @@
+module IntervalResponse
+  VERSION = "0.1.0"
+end

data/spec/interval_response/sequence_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+require 'complexity_assert'
+
+RSpec.describe IntervalResponse::Sequence do
+  context 'with a number of sized segments' do
+    it 'allows interval queries and yields tuples with the given object and the range inside it' do
+      seq = described_class.new
+      expect(seq.size).to eq(0)
+      expect(seq).to be_empty
+
+      a = double(:a, size: 6)
+      b = double(:b, size: 12)
+      c = double(:c, size: 17)
+      seq << a << b << c
+
+      expect(seq).not_to be_empty
+      expect(seq.size).to eq(6 + 12 + 17)
+      expect { |b|
+        seq.each_in_range(0..0, &b)
+      }.to yield_with_args(a, 0..0)
+
+      expect { |b|
+        seq.each_in_range(0..7, &b)
+      }.to yield_successive_args([a, 0..5], [b, 0..1])
+
+      expect { |b|
+        seq.each_in_range(7..27, &b)
+      }.to yield_successive_args([b, 1..11], [c, 0..9])
+
+      expect { |b|
+        seq.each_in_range(0..(6 + 12 - 1), &b)
+      }.to yield_successive_args([a, 0..5], [b, 0..11])
+    end
+
+    it 'generates the ETag for an empty sequence, and the etag contains data' do
+      seq = described_class.new
+      etag_for_sequence = seq.etag
+      expect(etag_for_sequence).to start_with('"')
+      expect(etag_for_sequence).to end_with('"')
+      expect(etag_for_sequence.bytesize).to be > 8
+    end
+
+    it 'generates the ETag dependent on the sequence composition' do
+      a = double(:a, size: 6)
+      b = double(:b, size: 12)
+      c = double(:c, size: 17)
+      seq = described_class.new(a, b, c)
+      etag_for_sequence = seq.etag
+      expect(etag_for_sequence).to start_with('"')
+      expect(etag_for_sequence).to end_with('"')
+
+      seq = described_class.new(a, b, c)
+      etag_for_sequence_of_same_sizes = seq.etag
+      expect(etag_for_sequence_of_same_sizes).to eq(etag_for_sequence)
+
+      seq = described_class.new(a, b, double(size: 7))
+      etag_for_sequence_of_same_sizes = seq.etag
+      expect(etag_for_sequence_of_same_sizes).not_to eq(etag_for_sequence)
+    end
+
+    it 'can handle a range that stretches outside of the available range' do
+      a = double('a', size: 3)
+      b = double('b', size: 4)
+      c = double('c', size: 1)
+
+      seq = described_class.new(a, b, c)
+      expect { |b|
+        seq.each_in_range(0..27, &b)
+      }.to yield_successive_args([a, 0..2], [b, 0..3], [c, 0..0])
+    end
+
+    it 'is composable' do
+      a = double('a', size: 3)
+      b = double('b', size: 4)
+      c = double('c', size: 1)
+
+      seq = described_class.new(a, b, described_class.new(c))
+
+      expect { |b|
+        seq.each_in_range(0..27, &b)
+      }.to yield_successive_args([a, 0..2], [b, 0..3], [c, 0..0])
+    end
+
+    it 'has close to linear performance with large number of ranges and intervals' do
+      module RangeIntervalCombinedComplexity
+        ONE_SEGMENT = Struct.new(:size).new(13)
+        def self.generate_args(size)
+          intervals = [ONE_SEGMENT] * size
+          seq = IntervalResponse::Sequence.new(*intervals)
+          http_ranges = size.times.map do |n|
+            range_start = (n * 13) + 4
+            range_end = (n * 13) + 12
+            range_start..range_end
+          end
+
+          [seq, http_ranges]
+        end
+
+        def self.run(seq, http_ranges)
+          http_ranges.each do |r|
+            seq.each_in_range(r) do |double, range|
+              # pass
+            end
+          end
+        end
+      end
+      expect(RangeIntervalCombinedComplexity).to be_linear
+    end
+
+    it 'has close to linear performance with a range in the middle' do
+      module SearchInMiddle
+        ONE_SEGMENT = Struct.new(:size).new(13)
+        def self.generate_args(size)
+          intervals = [ONE_SEGMENT] * size
+          seq = IntervalResponse::Sequence.new(*intervals)
+          range_start = ((size / 2) * 13) + 4
+          range_end = ((size / 2) * 13) + 128
+
+          [seq, range_start..range_end]
+        end
+
+        def self.run(seq, r)
+          seq.each_in_range(r) do |double, range|
+            # pass
+          end
+        end
+      end
+      expect(SearchInMiddle).to be_linear
+    end
+  end
+end

data/spec/interval_response_spec.rb

@@ -0,0 +1,156 @@
+RSpec.describe IntervalResponse do
+  it "has a version number" do
+    expect(IntervalResponse::VERSION).not_to be nil
+  end
+
+  context 'with an empty resource' do
+    let(:seq) { IntervalResponse::Sequence.new }
+    it 'always returns the empty response' do
+      response = IntervalResponse.new(seq, _http_range_header = nil, _if_range_header = nil)
+      expect(response.status_code).to eq(200)
+      expect(response.content_length).to eq(0)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "0",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+      expect { |b|
+        response.each(&b)
+      }.not_to yield_control
+
+      response = IntervalResponse.new(seq, 'bytes=0-', _if_range = nil)
+      expect(response.status_code).to eq(200)
+      expect(response.content_length).to eq(0)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "0",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+      expect { |b|
+        response.each(&b)
+      }.not_to yield_control
+    end
+  end
+
+  context 'with intervals containing data' do
+    let(:segment_a) { 'yes' }
+    let(:segment_b) { ' we ' }
+    let(:segment_c) { '!' }
+
+    let(:seq) do
+      IntervalResponse::Sequence.new(segment_a, segment_b, segment_c)
+    end
+
+    it 'returns the full response if the client did not ask for a Range' do
+      response = IntervalResponse.new(seq, _http_range_header = nil, _if_range = nil)
+      expect(response.status_code).to eq(200)
+      expect(response.content_length).to eq(3 + 4 + 1)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "8",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+      expect { |b|
+        response.each(&b)
+      }.to yield_successive_args([segment_a, 0..2], [segment_b, 0..3], [segment_c, 0..0])
+    end
+
+    it 'returns a single HTTP range if the client asked for it and it can be satisfied' do
+      response = IntervalResponse.new(seq, "bytes=2-4", _if_range = nil)
+      expect(response.status_code).to eq(206)
+      expect(response.content_length).to eq(3)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "3",
+        "Content-Range" => "bytes 2-4/8",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+      expect { |b|
+        response.each(&b)
+      }.to yield_successive_args([segment_a, 2..2], [segment_b, 0..1])
+    end
+
+    it 'returns a single HTTP range if the client asked for it and it can be satisfied, ETag matches' do
+      response = IntervalResponse.new(seq, "bytes=2-4", _if_range = seq.etag)
+      expect(response.status_code).to eq(206)
+      expect(response.content_length).to eq(3)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "3",
+        "Content-Range" => "bytes 2-4/8",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+      expect { |b|
+        response.each(&b)
+      }.to yield_successive_args([segment_a, 2..2], [segment_b, 0..1])
+    end
+
+    it 'responss with the entier resource if the Range is satisfiable but the If-Range specifies a different ETag than the sequence' do
+      response = IntervalResponse.new(seq, _http_range_header = "bytes=12901-", _http_if_range = '"different"')
+      expect(response.status_code).to eq(200)
+      expect(response.content_length).to eq(8)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "8",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+    end
+
+    it 'responds with the range that can be satisfied if asked for 2 of which one is unsatisfiable' do
+      response = IntervalResponse.new(seq, _http_range_header = "bytes=0-5,12901-", _http_if_range = nil)
+      expect(response.status_code).to eq(206)
+      expect(response.content_length).to eq(6)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "6",
+        "Content-Range" => "bytes 0-5/8",
+        "Content-Type" => "binary/octet-stream",
+        'ETag' => seq.etag,
+      )
+
+      expect { |b|
+        response.each(&b)
+      }.to yield_successive_args([segment_a, 0..2], [segment_b, 0..2])
+    end
+
+    it 'responds with MIME multipart of ranges if the client asked for it and it can be satisfied' do
+      response = IntervalResponse.new(seq, _http_range_header = "bytes=0-0,2-2", _http_if_range = nil)
+      response.instance_variable_set('@boundary', 'tcROXEYMdRNXRRYstW296yM1')
+
+      expect(response.status_code).to eq(206)
+      expect(response.content_length).to eq(190)
+      expect(response.headers).to eq(
+        "Accept-Ranges" => "bytes",
+        "Content-Length" => "190",
+        "Content-Type" => "multipart/byte-ranges; boundary=tcROXEYMdRNXRRYstW296yM1",
+        'ETag' => seq.etag,
+      )
+
+      output = StringIO.new
+      response.each do |segment, range|
+        output.write(segment[range])
+      end
+
+      reference = [
+        "--tcROXEYMdRNXRRYstW296yM1\r\n",
+        "Content-Type: binary/octet-stream\r\n",
+        "Content-Range: bytes 0-0/8\r\n",
+        "\r\n",
+        "y\r\n",
+        "--tcROXEYMdRNXRRYstW296yM1\r\n",
+        "Content-Type: binary/octet-stream\r\n",
+        "Content-Range: bytes 2-2/8\r\n",
+        "\r\n",
+        "s",
+      ].join
+      expect(output.string).to eq(reference)
+      expect(output.string.bytesize).to eq(190)
+    end
+  end
+end

data/spec/rack_stack_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+require 'rack/test'
+
+RSpec.describe 'IntervalResponse used in a Rack application' do
+  include Rack::Test::Methods
+  let(:segments) { @segments || [] }
+  let(:app) do
+    ->(env) {
+      interval_sequence = IntervalResponse::Sequence.new(*segments)
+      response = IntervalResponse.new(interval_sequence, env['HTTP_RANGE'], env['HTTP_IF_RANGE'])
+      response.to_rack_response_triplet
+    }
+  end
+
+  def tempfile_with_random_bytes(n_bytes)
+    Tempfile.new('segment').tap do |tf|
+      tf.write(Random.new.bytes(n_bytes))
+      tf.flush
+      tf.rewind
+    end
+  end
+
+  it 'returns a full response via the Rack adapter' do
+    @segments = ["Mary", " had", " a little", " lamb"]
+    get '/words'
+    expect(last_response).to be_ok
+    expect(last_response.body).to eq("Mary had a little lamb")
+  end
+
+  it 'returns a Range response via the Rack adapter' do
+    @segments = ["Mary", " had", " a little", " lamb"]
+    get '/words', nil, 'HTTP_RANGE' => 'bytes=1-5'
+    expect(last_response.status).to eq(206)
+    expect(last_response.content_length).to eq(5)
+    expect(last_response.body).to eq("ary h")
+  end
+
+  it 'serves from large-ish files' do
+    tiny = "tiny string"
+    file_a = tempfile_with_random_bytes(4 * 1024 * 1024)
+    file_b = tempfile_with_random_bytes(7 * 1024 * 1024)
+
+    @segments = [tiny, file_a, file_b]
+    get '/big', nil, 'HTTP_RANGE' => 'bytes=1-5'
+    expect(last_response.status).to eq(206)
+    expect(last_response.content_length).to eq(5)
+
+    get '/big', nil, 'HTTP_RANGE' => 'bytes=2-56898'
+    expect(last_response.status).to eq(206)
+    expect(last_response.content_length).to eq(56897)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,14 @@
+require "bundler/setup"
+require "interval_response"
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = ".rspec_status"
+
+  # Disable RSpec exposing methods globally on `Module` and `main`
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

metadata

@@ -0,0 +1,183 @@
+--- !ruby/object:Gem::Specification
+name: interval_response
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Julik Tarkhanov
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-09-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: measurometer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: complexity_assert
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: wetransfer_style
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Assemble HTTP responses from spliced sequences of payloads
+email:
+- me@julik.nl
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- interval_response.gemspec
+- lib/interval_response.rb
+- lib/interval_response/empty.rb
+- lib/interval_response/full.rb
+- lib/interval_response/invalid.rb
+- lib/interval_response/lazy_file.rb
+- lib/interval_response/multi.rb
+- lib/interval_response/sequence.rb
+- lib/interval_response/single.rb
+- lib/interval_response/to_rack_response_triplet.rb
+- lib/interval_response/version.rb
+- spec/interval_response/sequence_spec.rb
+- spec/interval_response_spec.rb
+- spec/rack_stack_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/WeTransfer/interval_response
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/WeTransfer/interval_response
+  source_code_uri: https://github.com/WeTransfer/interval_response
+  changelog_uri: https://github.com/WeTransfer/interval_response
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Assemble HTTP responses from spliced sequences of payloads
+test_files: []