regurgitator 0.0.0 → 0.1.0

data/ChangeLog

@@ -1,5 +1,69 @@
 ChangeLog from http://bogomips.org/regurgitator.git
 
+   commit 84e48c63074049f32660672b5a3c195d12dc3896
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Nov 11 08:46:12 2011 +0000
+   
+       regurgitator 0.1.0
+       
+       * local_file: support HTTP Range: requests
+       * minor code cleanups
+       * update/fix install dependencies (include rpatricia)
+   
+   commit 341b4d3198013893ae9929d588cfde73fa39b2ee
+   Author: Eric Wong <e@yhbt.net>
+   Date:   Fri Nov 11 08:33:40 2011 +0000
+   
+       local_file: support HTTP Range: requests
+       
+       This should give local users Range: functionality since most
+       HTTP servers used for MogileFS storage support Range:, too.
+       For requests that require proxying back to the HTTP server,
+       this we've always forwarded Range: requests and Content-Range:
+       responses as-is in a completely agnostic manner.
+   
+   commit a898550de8398fafef9690f29568fffe351df68b
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Nov 11 08:22:58 2011 +0000
+   
+       bump dependencies
+       
+       This forces newer kgio and newer http_spew on people, and
+       we'll need the latest rack for Range: parsing.
+   
+   commit 6cb4d49092eceee6720a702ca51f96851711b612
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Oct 21 17:41:03 2011 -0700
+   
+       use raise instead of abort in case we hit bugs
+       
+       These should probably nuke the process, but we'll be nice
+       and raise instead since we shouldn't cause data corruption.
+   
+   commit 70f8be5607ecdf087e20c0f539e6cac9c8b6bc16
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Jun 24 18:00:01 2011 -0700
+   
+       test_local: use Dir.mktmpdir
+       
+       The "tmpdir" module in the standhas a handy method for creating tmpdirs
+   
+   commit 752fdf60e8ac560d07a0e2f7d3ee58f075c620b9
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Jun 24 17:56:38 2011 -0700
+   
+       less initialize/super magic
+       
+       It's making things too hard to keep track of
+   
+   commit 7d826aae4753eb9eeca3467accb2c13eabd4a8eb
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Wed May 18 16:38:51 2011 -0700
+   
+       gemspec: add rpatricia dependency
+       
+       Not everybody has this gem installed by default :x
+   
    commit 0104cfec9974cc65d79ce614ab37ba3fbd672b11
    Author: Eric Wong <normalperson@yhbt.net>
    Date:   Tue May 10 15:09:50 2011 -0700

data/GIT-VERSION-FILE

@@ -1 +1 @@
-GIT_VERSION = 0.0.0
+GIT_VERSION = 0.1.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.0.0.GIT
+DEF_VER=v0.1.0.GIT
 
 LF='
 '

data/LATEST

@@ -1 +1,6 @@
-Currently unreleased
+=== regurgitator 0.1.0 / 2011-11-11 08:47 UTC
+
+  * local_file: support HTTP Range: requests
+  * minor code cleanups
+  * update/fix install dependencies (include rpatricia)
+

data/NEWS

@@ -1 +1,6 @@
-No news yet.
+=== regurgitator 0.1.0 / 2011-11-11 08:47 UTC
+
+  * local_file: support HTTP Range: requests
+  * minor code cleanups
+  * update/fix install dependencies (include rpatricia)
+

data/lib/regurgitator/device.rb

@@ -14,19 +14,15 @@ module Regurgitator::Device
   DEVICES_ON = 'SELECT devid FROM file_on WHERE fid = ?'
   # :startdoc:
 
-  def initialize(*args) # :nodoc:
-    super
-    init_device_ivars
-  end
-
-  def init_device_ivars
+  def device_init
+    server_settings_init
     @device_cache_mtime = Time.at(0)
     @device_cache = nil
     @device_cache_lock = Mutex.new
   end
 
   def self.extended(obj)
-    obj.init_device_ivars
+    obj.device_init
   end
 
   def device_uri_pair!(opts, get_port)

data/lib/regurgitator/domain.rb

@@ -4,8 +4,7 @@ module Regurgitator::Domain
 
   REFRESH_DOMAIN = 'SELECT dmid,namespace FROM domain' # :nodoc:
 
-  def initialize(*args) # :nodoc:
-    super
+  def domain_init
     @domain_lock = Mutex.new
     @domain_cache_mtime = Time.at(0)
     @domain_cache = nil

data/lib/regurgitator/domain_host.rb

@@ -27,7 +27,7 @@ class Regurgitator::DomainHost
       raise TypeError, ":suffix must be a String or Regexp #{suffix.inspect}"
     end
     @domain_regexp = suffix
-    super(app, opts[:db])
+    endpoint_init(app, opts[:db])
   end
 
   def call(env) # :nodoc:

data/lib/regurgitator/domain_path.rb

@@ -28,6 +28,6 @@ class Regurgitator::DomainPath
   end
 
   def initialize(app, db) # :nodoc:
-    super(app, Hash === db ? db[:db] : db)
+    endpoint_init(app, Hash === db ? db[:db] : db)
   end
 end

data/lib/regurgitator/endpoint.rb

@@ -7,10 +7,10 @@ module Regurgitator::Endpoint
 
   REPROXY_KEY = Regurgitator::REPROXY_KEY # :nodoc:
 
-  def initialize(app, db)
+  def endpoint_init(app, db)
     @app = app
     @db = db
-    super
+    file_info_init
   end
 
   def best_addr(env)
@@ -87,7 +87,7 @@ module Regurgitator::Endpoint
         body.close if body.respond_to?(:close)
         return [ status, headers, [] ]
       else
-        abort "Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
+        raise "BUG: Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
       end
     rescue Regurgitator::NoDevices
       retry unless zone_uris.empty?

data/lib/regurgitator/file_info.rb

@@ -5,6 +5,11 @@ module Regurgitator::FileInfo
 
   # :stopdoc:
   FILE_INFO = 'SELECT fid,length FROM file WHERE dmid = ? AND dkey = ? LIMIT 1'
+
+  def file_info_init
+    domain_init
+    device_init
+  end
   # :startdoc:
 
   # returns a hash with file information and URIs for accessing the file:

data/lib/regurgitator/local.rb

@@ -55,7 +55,7 @@ module Regurgitator::Local
     Regurgitator::Local.include?(uri.host) or return
     (roots = STORE_PATHS[uri.port]).empty? and return
     uri.path =~ %r{\A/(dev\d+)/} or
-            abort "BUG: path needs to match '\\A/dev\d+/' (#{uri})"
+            raise "BUG: path needs to match '\\A/dev\d+/' (#{uri})"
     devN = $1
     rv = nil
     roots.each do |root, dev_dirs|

data/lib/regurgitator/local_file.rb

@@ -18,12 +18,30 @@ class Regurgitator::LocalFile
     if modified_since = env["HTTP_IF_MODIFIED_SINCE"]
       modified_since?(modified_since, stat) or return
     end
+    size = stat.size
+
+    headers = {
+      "Content-Type" => "application/octet-stream", # always ".fid"
+      "Content-Length" => size.to_s,
+      "Last-Modified" => stat.mtime.httpdate,
+      "Accept-Ranges" => "bytes",
+    }
+    @response = [ 200, headers ]
+
+    ranges = Rack::Utils.byte_ranges(env, size)
+    if nil == ranges || ranges.size > 1
+      @range = nil
+    elsif @range = ranges[0]
+      @response[0] = 206
+      headers["Content-Range"] = "bytes #{@range.begin}-#{@range.end}/#{size}"
+      headers["Content-Length"] = (@range.end - @range.begin + 1).to_s
+    else
+      @response[0] = 416
+      headers["Content-Range"] = "bytes */#{size}"
+      @response << []
+      return
+    end
 
-    @response = [ 200, {
-        "Content-Type" => "application/octet-stream", # always ".fid"
-        "Content-Length" => stat.size.to_s,
-        "Last-Modified" => stat.mtime.httpdate,
-    } ]
     case env["REQUEST_METHOD"]
     when "GET"
       @to_path, @uri = path, uri
@@ -31,17 +49,27 @@ class Regurgitator::LocalFile
     when "HEAD"
       @response << []
     else
-      abort "Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
+      raise "BUG: Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
     end
   end
 
   # normal Rack HTTP server endpoint, used if the server can't handle
   # +to_path+
   def each
-    buf = ""
     File.open(@to_path) do |fp|
-      while fp.read(0x4000, buf)
-        yield buf
+      buf = ""
+      max = 0x4000
+      if @range
+        fp.seek(@range.begin)
+        len = @range.end - @range.begin + 1
+        while len > 0 && fp.read(len > max ? max : len, buf)
+          len -= buf.size
+          yield buf
+        end
+      else
+        while fp.read(0x4000, buf)
+          yield buf
+        end
       end
     end
   end

data/lib/regurgitator/one_domain.rb

@@ -19,7 +19,7 @@ class Regurgitator::OneDomain
 
   def initialize(app, opts) # :nodoc:
     @domain = opts[:domain]
-    super(app, opts[:db])
+    endpoint_init(app, opts[:db])
   end
 
   def call(env) # :nodoc:

data/lib/regurgitator/server_settings.rb

@@ -6,19 +6,14 @@ module Regurgitator::ServerSettings
   SETTINGS_LOOKUP = 'SELECT value FROM server_settings WHERE field = ? LIMIT 1'
   # :startdoc:
 
-  def initialize(*args) # :nodoc:
-    super
-    init_zone_ivars
-  end
-
-  def init_zone_ivars
+  def server_settings_init
     @zone_cache_mtime = Time.at(0)
     @zone_cache = nil
     @zone_cache_lock = Mutex.new
   end
 
   def self.extended(obj)
-    obj.init_zone_ivars
+    obj.server_settings_init
   end
 
   def refresh_zone(force = false) # :nodoc:

data/regurgitator.gemspec

@@ -21,10 +21,11 @@ Gem::Specification.new do |s|
   s.rubyforge_project = %q{rainbows}
   s.test_files = Dir["test/test_*.rb"]
 
-  s.add_dependency("rack", ["~> 1.2"])
-  s.add_dependency("sequel", ["~> 3.23"])
-  s.add_dependency("http_spew", ["~> 0.1"])
+  s.add_dependency("rack", ["~> 1.3"])
+  s.add_dependency("sequel", ["~> 3.29"])
+  s.add_dependency("http_spew", ["~> 0.4"])
+  s.add_dependency("rpatricia", ["~> 0.08"])
 
-  s.add_development_dependency('wrongdoc', "~> 1.5")
+  s.add_development_dependency('wrongdoc', "~> 1.6")
   # s.licenses = %w(GPL2 GPL3) # accessor not compatible with older RubyGems
 end

data/test/test_local.rb

@@ -1,6 +1,6 @@
 require "test/unit"
 require "regurgitator"
-require "tempfile"
+require "tmpdir"
 require "fileutils"
 
 class TestLocal < Test::Unit::TestCase
@@ -16,10 +16,7 @@ class TestLocal < Test::Unit::TestCase
     @local_addrs = Regurgitator::Local.instance_variable_get(:@local_addrs)
     @local_addrs["127.0.0.1"] = true if @local_addrs.empty?
     @addr = @local_addrs.keys.first
-    tmp = Tempfile.new('test_local')
-    @tmpdir = tmp.path
-    tmp.close!
-    Dir.mkdir(@tmpdir)
+    @tmpdir = Dir.mktmpdir
   end
 
   def test_device_path_stat
@@ -74,5 +71,42 @@ class TestLocal < Test::Unit::TestCase
     File.open(path_a, "wb") { |fp| fp.write '.' }
     res = trylocal(env, [ [ a, b ] ])
     assert_kind_of Regurgitator::LocalFile, res
+    assert_equal 200, res.response[0]
+  end
+
+  def test_trylocal_range
+    port = 1233
+    env = { "REQUEST_METHOD" => "GET", "HTTP_RANGE" => "bytes=5-15" }
+    a = URI("http://#@addr:#{port}/dev10/0/004/689/0004689631.fid")
+    b = URI("http://#@addr:#{port + 6}/dev11/0/004/689/0004689631.fid")
+    assert_nil trylocal(env, [ [ a, b ] ])
+
+    path_a = "#@tmpdir#{a.path}"
+    FileUtils.mkdir_p(File.dirname(path_a))
+    Regurgitator::Local.register port, @tmpdir
+    File.open(path_a, "wb") { |fp| (0..15).each { |i| fp.write(i.to_s(16)) } }
+    res = trylocal(env, [ [ a, b ] ])
+    assert_kind_of Regurgitator::LocalFile, res
+    assert_equal(206, res.response[0])
+    body = ""
+    res.response[2].each { |x| body << x }
+    expect = (5..15).map { |x| x.to_s(16) }.join("")
+    assert_equal expect, body
+  end
+
+  def test_trylocal_bad_range
+    port = 1233
+    env = { "REQUEST_METHOD" => "GET", "HTTP_RANGE" => "bytes=515-999" }
+    a = URI("http://#@addr:#{port}/dev10/0/004/689/0004689631.fid")
+    b = URI("http://#@addr:#{port + 6}/dev11/0/004/689/0004689631.fid")
+    assert_nil trylocal(env, [ [ a, b ] ])
+
+    path_a = "#@tmpdir#{a.path}"
+    FileUtils.mkdir_p(File.dirname(path_a))
+    Regurgitator::Local.register port, @tmpdir
+    File.open(path_a, "wb") { |fp| (0..15).each { |i| fp.write(i.to_s(16)) } }
+    res = trylocal(env, [ [ a, b ] ])
+    assert_kind_of Regurgitator::LocalFile, res
+    assert_equal(416, res.response[0])
   end
 end

data/test/test_server_settings.rb

@@ -12,6 +12,7 @@ class TestServerSettings < Test::Unit::TestCase
       String :field, :size => 50, :null => false, :primary_key => true
       String :value, :size => 255
     end
+    server_settings_init
   end
 
   def test_zones

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: regurgitator
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 0
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Regurgitators
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-10 00:00:00 Z
+date: 2011-11-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rack
@@ -25,11 +25,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 11
+        hash: 9
         segments: 
         - 1
-        - 2
-        version: "1.2"
+        - 3
+        version: "1.3"
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
@@ -40,11 +40,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 41
+        hash: 61
         segments: 
         - 3
-        - 23
-        version: "3.23"
+        - 29
+        version: "3.29"
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -55,28 +55,43 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 9
+        hash: 3
         segments: 
         - 0
-        - 1
-        version: "0.1"
+        - 4
+        version: "0.4"
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: wrongdoc
+  name: rpatricia
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 27
+        segments: 
+        - 0
+        - 8
+        version: "0.08"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: wrongdoc
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 1
-        - 5
-        version: "1.5"
+        - 6
+        version: "1.6"
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 description: |-
   \Regurgitator is an GPL-licensed library and Rack middleware for
   serving files stored in MogileFS.  It can be embedded inside
@@ -182,7 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: rainbows
-rubygems_version: 1.8.1
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: read-only Rack endpoints for MogileFS