regurgitator 0.0.0

data/GIT-VERSION-FILE

@@ -0,0 +1 @@
+GIT_VERSION = 0.0.0

data/GIT-VERSION-GEN

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+GVF=GIT-VERSION-FILE
+DEF_VER=v0.0.0.GIT
+
+LF='
+'
+
+# First see if there is a version file (included in release tarballs),
+# then try git-describe, then default.
+if test -f version
+then
+	VN=$(cat version) || VN="$DEF_VER"
+elif test -d .git -o -f .git &&
+	VN=$(git describe --abbrev=4 HEAD 2>/dev/null) &&
+	case "$VN" in
+	*$LF*) (exit 1) ;;
+	v[0-9]*)
+		git update-index -q --refresh
+		test -z "$(git diff-index --name-only HEAD --)" ||
+		VN="$VN-dirty" ;;
+	esac
+then
+	VN=$(echo "$VN" | sed -e 's/-/./g');
+else
+	VN="$DEF_VER"
+fi
+
+VN=$(expr "$VN" : v*'\(.*\)')
+
+if test -r $GVF
+then
+	VC=$(sed -e 's/^GIT_VERSION = //' <$GVF)
+else
+	VC=unset
+fi
+test "$VN" = "$VC" || {
+	echo >&2 "GIT_VERSION = $VN"
+	echo "GIT_VERSION = $VN" >$GVF
+}

data/GNUmakefile

@@ -0,0 +1,8 @@
+all::
+RSYNC_DEST := bogomips.org:/srv/bogomips/regurgitator
+rfproject := rainbows
+rfpackage := regurgitator
+include pkg.mk
+all:: test-integration
+test-integration:
+	$(MAKE) -C t

data/LATEST

@@ -0,0 +1 @@
+Currently unreleased

data/LICENSE

@@ -0,0 +1,18 @@
+regurgitator is copyrighted Free Software by all contributors, see logs
+in revision control for names and email addresses of all of them.
+
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, version {2}[http://www.gnu.org/licenses/gpl-2.0.txt] or
+or {3}[http://www.gnu.org/licenses/gpl-3.0.txt] (see link:COPYING).
+The regurgitator project leader (Eric Wong) reserves the right to
+relicense regurgitator under future versions of the GPL.
+
+regurgitator is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this library; if not, write to the Free Software Foundation,
+Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301

data/NEWS

@@ -0,0 +1 @@
+No news yet.

data/README

@@ -0,0 +1,49 @@
+= \Regurgitator - read-only Rack endpoints for MogileFS
+
+\Regurgitator is an GPL-licensed library and Rack middleware for
+serving files stored in MogileFS.  It can be embedded inside
+any existing Rack application or be used as a standalone Rack app.
+
+\Regurgitator talks directly to the underlying RDBMS powering MogileFS
+to avoid extra layers and latency.  \Regurgitator only needs
+read-only access to the RDBMS and can connect to slave databases.
+
+== Rack Endpoints
+
+\Regurgitator comes with your choice of three standard Rack endpoints for
+serving files stored in MogileFS.  Pick the one best suited for your
+application:
+
+* {DomainPath}[link:Regurgitator/DomainPath.html]
+* {DomainHost}[link:Regurgitator/DomainHost.html]
+* {OneDomain}[link:Regurgitator/OneDomain.html]
+
+It should also be easy to roll your own based on your needs by reusing
+our library code.
+
+== Hacking
+
+You can get the latest source via git from the following locations:
+
+  git://bogomips.org/regurgitator.git
+  git://repo.or.cz/regurgitator.git (mirror)
+
+You may browse the code from the web and download the latest snapshot
+tarballs here:
+
+* http://bogomips.org/regurgitator.git (cgit)
+* http://repo.or.cz/w/regurgitator.git (gitweb)
+
+Inline patches (from "git format-patch") to the mailing list are
+preferred because they allow code review and comments in the reply to
+the patch.
+
+We will adhere to mostly the same conventions for patch submissions as
+git itself.  See the Documentation/SubmittingPatches document
+distributed with git on on patch submission guidelines to follow.  Just
+don't email the git mailing list or maintainer with \Regurgitator patches.
+
+== Contact
+
+All feedback (bug reports, user/development discussion, patches, pull
+requests) go to the mailing list: mailto:barfs@librelist.org

data/examples/domain_host.ru

@@ -0,0 +1,6 @@
+require 'regurgitator'
+db = Sequel.connect 'mysql2://user@example.com/mogilefs'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+err = lambda { |env| [ 404, [], [] ] }
+run Regurgitator::DomainHost.new(err, :db => db, :suffix => '.example.com')

data/examples/domain_path.ru

@@ -0,0 +1,5 @@
+require 'regurgitator'
+db = Sequel.connect 'mysql2://user@example.com/mogilefs'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+run Regurgitator::DomainPath.new(lambda { |env| [ 404, [], [] ] }, db)

data/examples/one_domain.ru

@@ -0,0 +1,6 @@
+require 'regurgitator'
+db = Sequel.connect 'mysql2://user@example.com/mogilefs'
+use Rack::ContentLength
+use Rack::ContentType, 'text/plain'
+err = lambda { |env| [ 404, [], [] ] }
+run Regurgitator::OneDomain.new(err, :db => db, :domain => 'my.domain')

data/lib/regurgitator.rb

@@ -0,0 +1,43 @@
+# -*- encoding: binary -*-
+require 'uri'
+require 'rack'
+require 'thread'
+require 'sequel'
+require 'http_spew'
+require 'rpatricia'
+
+# The Regurgitator main module, serving as a namespace for all
+# modules and classes.
+#
+# All modules meant for use in applications are autoload-ed,
+# so just "require 'regurgitator'" in your code.
+module Regurgitator
+
+  # This HTTP header is read to determine a location to reproxy to.
+  # Add this header in your nginx config.
+  REPROXY_KEY = 'HTTP_X_REPROXY_PATH'
+
+  autoload :ServerSettings, 'regurgitator/server_settings'
+  autoload :Domain, 'regurgitator/domain'
+  autoload :Device, 'regurgitator/device'
+  autoload :FileInfo, 'regurgitator/file_info'
+  autoload :Endpoint, 'regurgitator/endpoint'
+  autoload :FileRequest, 'regurgitator/file_request'
+  autoload :Local, 'regurgitator/local'
+  autoload :LocalFile, 'regurgitator/local_file'
+
+  # Rack middlewares/apps
+  autoload :DomainPath, 'regurgitator/domain_path'
+  autoload :DomainHost, 'regurgitator/domain_host'
+  autoload :OneDomain, 'regurgitator/one_domain'
+
+  # used to wrap up all Regurgitator-specific extensions
+  class Error < StandardError; end
+
+  # raised when there are no readable devices
+  class NoDevices < Error; end
+
+  # raised by FileRequest when the HTTP status code
+  # is outside of (200-299, 304)
+  class BadResponse < Error; end
+end

data/lib/regurgitator/device.rb

@@ -0,0 +1,79 @@
+# -*- encoding: binary -*-
+# helpers for device lookups
+module Regurgitator::Device
+  include Regurgitator::ServerSettings
+
+  # :stopdoc:
+  REFRESH_DEVICE = <<-EOS
+    SELECT d.devid, h.hostip, h.altip, h.altmask, h.http_port, h.http_get_port
+    FROM device d
+      LEFT JOIN host h ON d.hostid = h.hostid
+    WHERE d.status IN ('readonly','alive','drain') AND h.status = 'alive'
+  EOS
+
+  DEVICES_ON = 'SELECT devid FROM file_on WHERE fid = ?'
+  # :startdoc:
+
+  def initialize(*args) # :nodoc:
+    super
+    init_device_ivars
+  end
+
+  def init_device_ivars
+    @device_cache_mtime = Time.at(0)
+    @device_cache = nil
+    @device_cache_lock = Mutex.new
+  end
+
+  def self.extended(obj)
+    obj.init_device_ivars
+  end
+
+  def device_uri_pair!(opts, get_port)
+    rv = [ URI::HTTP.build(opts) ]
+    if get_port
+      opts[:port] = get_port
+      rv << URI::HTTP.build(opts)
+    end
+    rv
+  end
+
+  # Returns a hash of device info with the Integer +devid+
+  # as the hash key.
+  def refresh_device(force = false) # :nodoc:
+    @device_cache_lock.synchronize { refresh_device_unlocked(force) }
+  end
+
+  def refresh_device_unlocked(force) # :nodoc:
+    return @device_cache if ! force && ((Time.now - @device_cache_mtime) < 60)
+    tmp = {}.compare_by_identity
+    refresh_zone(force)
+    @db[REFRESH_DEVICE].each do |x|
+      # devices in "drain" status may hit raciness try those as a last resort
+      x[:preferred] = !!(x[:d_status] =~ %r{\A(?:readonly|alive)\z})
+      hostip = x[:hostip]
+      port = x[:http_port] || 80
+      get_port = x[:http_get_port]
+      x[:ipaddr] = { :hostip => hostip }
+      x[:zone] = zone_for(hostip)
+      devid = x[:devid]
+      o = { :path => "/dev#{devid}", :port => port, :host => hostip }
+
+      x[:uris] = { :pri => device_uri_pair!(o, get_port) }
+      x[:ipaddr][:altmask] = pat = Patricia.new
+      altmask = x[:altmask] and pat.add(altmask)
+
+      if altip = x[:altip]
+        x[:ipaddr][:altip] = altip
+        o[:host] = altip
+        o[:port] = port
+        x[:uris][:alt] = device_uri_pair!(o, get_port)
+      end
+
+      tmp[devid] = x
+    end
+    Regurgitator::Local.refresh_addrs!
+    @device_cache_mtime = Time.now
+    @device_cache = tmp
+  end
+end

data/lib/regurgitator/domain.rb

@@ -0,0 +1,37 @@
+# -*- encoding: binary -*-
+# helpers for domain lookups
+module Regurgitator::Domain
+
+  REFRESH_DOMAIN = 'SELECT dmid,namespace FROM domain' # :nodoc:
+
+  def initialize(*args) # :nodoc:
+    super
+    @domain_lock = Mutex.new
+    @domain_cache_mtime = Time.at(0)
+    @domain_cache = nil
+  end
+
+  # returns the +dmid+ (domain identifier/primary key) for a string +domain+
+  def get_dmid(domain)
+    rv = refresh_domain[domain] and return rv
+    return false if false == rv
+    rv = refresh_domain(true)[domain] and return rv
+    @domain_lock.synchronize { @domain_cache[domain] = false }
+  end
+
+  # We cache the list of all available domains in memory, this shouldn't
+  # be too huge, though...
+  #
+  # Returns a hash with string namespaces as keys and dmids as values
+  def refresh_domain(force = false) # :nodoc:
+    @domain_lock.synchronize { refresh_domain_unlocked(force) }
+  end
+
+  def refresh_domain_unlocked(force) # :nodoc:
+    return @domain_cache if ! force && ((Time.now - @domain_cache_mtime) < 60)
+    tmp = {}
+    @db[REFRESH_DOMAIN].each { |x| tmp[x[:namespace].freeze] = x[:dmid] }
+    @domain_cache_mtime = Time.now
+    @domain_cache = tmp
+  end
+end

data/lib/regurgitator/domain_host.rb

@@ -0,0 +1,43 @@
+# -*- encoding: binary -*-
+
+# Matches GET and HEAD requests in the format of ":domain.example.com/:dkey"
+# where domain is read from the Host: header.
+#
+# If a client were to make a request to "http://foo.example.com/bar",
+# this endpoint would respond with the file from the "foo" domain
+# with the key "bar".
+# To use as middleware:
+#    require 'regurgitator'
+#    db = Sequel.connect('mysql2://user@example.com/mogilefs')
+#    use Regurgitator::DomainHost, :suffix => '.example.com', :db => db
+#
+# See the {domain_host.ru}[link:examples/domain_host.ru]
+# example for a standalone app.
+
+class Regurgitator::DomainHost
+  include Regurgitator::Endpoint
+
+  def initialize(app, opts) # :nodoc:
+    case suffix = opts[:suffix]
+    when String
+      suffix = Regexp.quote(suffix)
+      suffix = %r{\A(.*)#{suffix}\z}
+    when Regexp
+    else
+      raise TypeError, ":suffix must be a String or Regexp #{suffix.inspect}"
+    end
+    @domain_regexp = suffix
+    super(app, opts[:db])
+  end
+
+  def call(env) # :nodoc:
+    case env['REQUEST_METHOD']
+    when 'GET', 'HEAD'
+      host = env['HTTP_HOST'] or return @app.call(env)
+      domain = @domain_regexp =~ host ? $1 : host
+      serve_file(env, domain, env['PATH_INFO'][1..-1])
+    else
+      @app.call(env)
+    end
+  end
+end

data/lib/regurgitator/domain_path.rb

@@ -0,0 +1,33 @@
+# -*- encoding: binary -*-
+
+# matches GET and HEAD requests in the format of "/:namespace/:dkey"
+# where +:dkey+ may contain multiple slashes
+#
+# If a client were to make a request to "http://example.com/d/foo/bar",
+# this endpoint would respond with the file from the "d" domain
+# with the key "foo/bar".
+#
+# To use as middleware:
+#    require 'regurgitator'
+#    db = Sequel.connect('mysql2://user@example.com/mogilefs')
+#    use Regurgitator::DomainPath, db
+#
+# See the {domain_path.ru}[link:examples/domain_host.ru]
+# example for a standalone app.
+class Regurgitator::DomainPath
+  include Regurgitator::Endpoint
+
+  def call(env) # :nodoc:
+    case env['REQUEST_METHOD']
+    when 'GET', 'HEAD'
+      env['PATH_INFO'] =~ %r{\A/([^/]+)/(.+)\z} or return @app.call(env)
+      serve_file(env, $1, $2)
+    else
+      @app.call(env)
+    end
+  end
+
+  def initialize(app, db) # :nodoc:
+    super(app, Hash === db ? db[:db] : db)
+  end
+end

data/lib/regurgitator/endpoint.rb

@@ -0,0 +1,97 @@
+# -*- encoding: binary -*-
+require "socket"
+module Regurgitator::Endpoint
+  include Regurgitator::FileInfo
+  include Rack::Utils
+  include Rack::Mime
+
+  REPROXY_KEY = Regurgitator::REPROXY_KEY # :nodoc:
+
+  def initialize(app, db)
+    @app = app
+    @db = db
+    super
+  end
+
+  def best_addr(env)
+    env.include?(REPROXY_KEY) ? env['REMOTE_ADDR'] : Regurgitator::Local.addr
+  end
+
+  # see nginx config examples for reproxy
+  def reproxy_path(env)
+    env[REPROXY_KEY]
+  end
+
+  # allow users to specify desired "Save As" filenames in the query string
+  def filename!(env, headers, dkey)
+    params = parse_query(env['QUERY_STRING'])
+    cd = nil
+    if fn = params['inline'] and escape(fn) == fn
+      cd, dkey = 'inline', fn
+    elsif fn = (params['attachment'] || params['filename']) and
+          escape(fn) == fn
+      cd, dkey = 'attachment', fn
+    end
+    headers['Content-Disposition'] = "#{cd}; filename=#{fn}" if cd
+    headers['Content-Type'] = mime_type(File.extname(dkey))
+  end
+
+  def empty_file(env, dkey)
+    headers = { 'Content-Length' => '0' }
+    filename!(env, headers, dkey)
+    [ 200, headers, [] ]
+  end
+
+  def serve_file(env, domain, dkey)
+    env['regurgitator.zone'] ||= best_addr(env)
+    info = file_info(env, domain, dkey) or return @app.call(env)
+    serve_info(env, info, dkey)
+  end
+
+  def serve_info(env, info, dkey)
+    info[:length] == 0 and return empty_file(env, dkey)
+    zone_uris = info[:uris]
+
+    zone = env['regurgitator.zone']
+    begin
+      uris = zone_uris.delete(zone)
+      uris ||= zone_uris.delete(zone_for(Regurgitator::Local.addr))
+
+      # no nearby zones? try all of them at once!
+      if uris.nil?
+        uris = []
+        zone_uris.each_value { |v| uris.concat v }
+        zone_uris.clear
+      end
+      uris or next
+
+      status, headers, body = r = Regurgitator::FileRequest.run(env, uris)
+      filename!(env, headers, dkey)
+      headers['ETag'] = %("#{info[:fid]}")
+
+      case env["REQUEST_METHOD"]
+      when "GET"
+        if body.respond_to?(:uri) && path = reproxy_path(env)
+          body.close if body.respond_to?(:close)
+          headers['Content-Length'] = '0'
+          headers['Location'] = body.uri.to_s
+          headers['X-Accel-Redirect'] = path
+          # yes we violate Rack::Lint here
+          %w(Content-Type Last-Modified).each do |key|
+            headers["X-Reproxy-#{key}"] = headers.delete(key)
+          end
+          return [ 302, headers, [] ]
+        end
+        return r
+      when "HEAD"
+        body.close if body.respond_to?(:close)
+        return [ status, headers, [] ]
+      else
+        abort "Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
+      end
+    rescue Regurgitator::NoDevices
+      retry unless zone_uris.empty?
+    end until zone_uris.empty?
+    @app.call(env)
+  end
+end