regurgitator 0.0.0

data/lib/regurgitator/file_info.rb

@@ -0,0 +1,53 @@
+# -*- encoding: binary -*-
+module Regurgitator::FileInfo
+  include Regurgitator::Domain
+  include Regurgitator::Device
+
+  # :stopdoc:
+  FILE_INFO = 'SELECT fid,length FROM file WHERE dmid = ? AND dkey = ? LIMIT 1'
+  # :startdoc:
+
+  # returns a hash with file information and URIs for accessing the file:
+  # {
+  #   :fid => 3149113,
+  #   :length => 17,
+  #   :uris => {
+  #     "east" => [ [read_write,read_only], [read_write,read_only] ],
+  #     "west" => [ [read_write,read_only], [read_write,read_only] ],
+  #     nil => [ [read_write,read_only], [read_write,read_only] ],
+  #   }
+  # }
+  def file_info(env, domain, dkey, update = false)
+    dmid = get_dmid(domain) or return
+    devices = refresh_device(update)
+    info = @db[FILE_INFO, dmid, dkey].first or return
+    0 == info[:length] and return info
+    fid = info[:fid]
+    read_uris = Hash.new { |h,k| h[k] = [] }
+    drain_uris = Hash.new { |h,k| h[k] = [] }
+
+    zone = env['regurgitator.zone']
+
+    @db[DEVICES_ON, fid].each do |x|
+      devinfo = devices[x[:devid]] or next
+      fid >= 10_000_000_000 and next # TODO: support larger FIDs
+
+      uris = devinfo[:uris][alt_zone?(zone, devinfo) ? :alt : :pri]
+      nfid = sprintf('%010u', fid)
+      /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
+      fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
+      uris = uris.map do |u|
+        u = u.dup
+        u.path += fid_path # must be a copy
+        u
+      end
+      (devinfo[:preferred] ? read_uris : drain_uris)[devinfo[:zone]] << uris
+    end
+    uris = info[:uris] = read_uris.empty? ? drain_uris : read_uris
+    (uris.empty? && ! update) ? file_info(env, domain, dkey, true) : info
+  end
+
+  def alt_zone?(zone, devinfo)
+    zone == :alt || devinfo[:ipaddr][:altmask].include?(zone)
+  end
+end

data/lib/regurgitator/file_request.rb

@@ -0,0 +1,48 @@
+# -*- encoding: binary -*-
+
+# Opens a IO.select-able object to read a (potentially large) file from a
+# backend.  We don't bother with keepalive since we want send buffers
+# to be empty, so make sure you have enough local ports.   You may want
+# to enable TIME-WAIT reuse/recycling in your TCP stack if your
+# network supports it.  This object is used as a Rack response
+# body.
+class Regurgitator::FileRequest < HTTP_Spew::Request
+
+  # used for doing redirects or reproxying with nginx
+  attr_reader :uri
+
+  extend Regurgitator::Local
+
+  # :stopdoc:
+  USER_AGENT = name.dup
+  PASS_HEADERS = %w(Range If-Modified-Since Accept-Encoding)
+  PASS_HEADERS.map! { |x| "HTTP_#{x.tr!('a-z-', 'A-Z_')}".freeze }
+  # :startdoc:
+
+  # +uris+ may be an array, read-only URIs are at the end
+  def initialize(env, uris)
+    uri = uris[-1] # a subclass of this may call multiple URIs
+    addr = Socket.pack_sockaddr_in(uri.port, uri.host)
+    req = {
+      "REQUEST_URI" => uri.request_uri,
+      "HTTP_HOST" => "#{uri.host}:#{uri.port}",
+    }.merge!(env)
+    @uri = uri
+    super req, nil, addr
+  end
+
+  def self.run(env, uri_group, timeout = env["regurgitator.timeout"] || 5)
+    local_file = trylocal(env, uri_group) and return local_file.response
+
+    req = {
+      "REQUEST_METHOD" => env["REQUEST_METHOD"],
+      "HTTP_USER_AGENT" => USER_AGENT,
+    }
+    PASS_HEADERS.each { |k| pass = env[k] and req[k] = pass }
+    tmp = HTTP_Spew.wait(1, uri_group.map { |uris| new(req, uris) }, timeout)
+    tmp.delete_if { |t| t.error }
+    ready = tmp.shift or raise Regurgitator::NoDevices, "no readable devices"
+    tmp.each { |t| t.close }
+    ready.response
+  end
+end

data/lib/regurgitator/local.rb

@@ -0,0 +1,89 @@
+# -*- encoding: binary -*-
+require "socket"
+require "set"
+
+module Regurgitator::Local
+
+  # This is used to register local storage endpoints so we can
+  # short-circuit and avoid making HTTP requests to devices
+  # Must be configured by the user
+  STORE_PATHS = Hash.new do |h,port|
+    h[port] = {} # key: directory root path, value: whatever...
+  end.compare_by_identity
+
+  @local_addrs = {}
+  @local_addrs_lock = Mutex.new
+
+  def self.include?(addr)
+    @local_addrs_lock.synchronize { @local_addrs.include?(addr) }
+  end
+
+  def self.addr
+    (@local_addrs_lock.synchronize { @local_addrs.first })[0]
+  end
+
+  def self.addrs
+    @local_addrs_lock.synchronize { @local_addrs.keys }
+  end
+
+  # Normally not needed unless you dynamically bring up/down network devices.
+  # It may be useful to call this periodically or to integrate with
+  # some network device state monitoring system.
+  def self.refresh_addrs!
+    @local_addrs_lock.synchronize do
+      tmp = {}
+      Socket.ip_address_list.keep_if do |ip|
+        ip.ipv4? && ! ip.ipv4_multicast?
+      end.each { |ip| tmp[ip.ip_address.freeze] = true }
+      @local_addrs = tmp
+    end
+  end
+  refresh_addrs!
+
+  # registers a local path for a given +tcp_port+ and +directory_root+
+  def self.register(tcp_port, directory_root)
+    directory_root = directory_root.gsub(%r{/+\z}, "")
+    dev_dirs = Dir.foreach(directory_root).grep(/\Adev\d+\z/)
+    raise ArgumentError, 'no /dev\d+/ directories found' if dev_dirs.empty?
+    STORE_PATHS[tcp_port][directory_root] = Set.new(dev_dirs)
+  end
+
+  # returns +nil+ if nothing was found
+  # returns a path and associated File::Stat to the local FS if a
+  # matching path is possible
+  def device_path_stat(uri)
+    Regurgitator::Local.include?(uri.host) or return
+    (roots = STORE_PATHS[uri.port]).empty? and return
+    uri.path =~ %r{\A/(dev\d+)/} or
+            abort "BUG: path needs to match '\\A/dev\d+/' (#{uri})"
+    devN = $1
+    rv = nil
+    roots.each do |root, dev_dirs|
+      dev_dirs.include?(devN) or next
+      begin
+        path = "#{root}#{uri.path}"
+        stat = File.stat(path)
+
+        # multiple "/devN" paths for the same ports would be ambiguous,
+        # so we cannot optimize away the HTTP request for those
+        return nil if rv
+
+        rv = [ path, stat ]
+      rescue => e
+        warn "E: #{e.message}, root=#{root} failed for #{uri}"
+      end
+    end
+
+    rv
+  end
+
+  def trylocal(env, uri_group)
+    STORE_PATHS.empty? and return
+    uri_group.flatten.each do |uri|
+      path_stat = device_path_stat(uri) or next
+      path, stat = path_stat
+      return Regurgitator::LocalFile.new(env, path, uri, stat)
+    end
+    nil
+  end
+end

data/lib/regurgitator/local_file.rb

@@ -0,0 +1,61 @@
+# -*- encoding: binary -*-
+require "time"
+
+# used to serve local files, bypassing HTTP to the backend device
+class Regurgitator::LocalFile
+
+  # may be called by some Rack web servers to use sendfile(2)
+  attr_reader :to_path
+
+  # used to provide redirects to clients
+  attr_reader :uri
+
+  # Stores the Rack response (a 3-element Array) on success to simulate
+  # HTTP_Spew::Request objects
+  attr_reader :response
+
+  def initialize(env, path, uri, stat)
+    if modified_since = env["HTTP_IF_MODIFIED_SINCE"]
+      modified_since?(modified_since, stat) or return
+    end
+
+    @response = [ 200, {
+        "Content-Type" => "application/octet-stream", # always ".fid"
+        "Content-Length" => stat.size.to_s,
+        "Last-Modified" => stat.mtime.httpdate,
+    } ]
+    case env["REQUEST_METHOD"]
+    when "GET"
+      @to_path, @uri = path, uri
+      @response << self
+    when "HEAD"
+      @response << []
+    else
+      abort "Unexpected REQUEST_METHOD=#{env['REQUEST_METHOD']}"
+    end
+  end
+
+  # normal Rack HTTP server endpoint, used if the server can't handle
+  # +to_path+
+  def each
+    buf = ""
+    File.open(@to_path) do |fp|
+      while fp.read(0x4000, buf)
+        yield buf
+      end
+    end
+  end
+
+  def modified_since?(modified_since, stat)
+    begin
+      modified_since = Time.httpdate(modified_since)
+    rescue ArgumentError
+      h = { "Content-Type" => "text/plain", "Content-Length" => "0" }
+      @response = [ 400, h, [] ]
+      return false
+    end
+    stat.mtime > modified_since and return true
+    @response = [ 304, {}, [] ]
+    false
+  end
+end

data/lib/regurgitator/one_domain.rb

@@ -0,0 +1,33 @@
+# -*- encoding: binary -*-
+
+# matches GET and HEAD requests in the format of "/:dkey" from a
+# single, preconfigured domain.
+#
+# If a client were to make a request to "http://example.com/bar",
+# this endpoint would respond with the file with the key "bar"
+# from the preconfigured domain.
+#
+# To use as middleware:
+#    require 'regurgitator'
+#    db = Sequel.connect('mysql2://user@example.com/mogilefs')
+#    use Regurgitator::OneDomain, :domain => 'foo', :db => db
+#
+# See the {one_domain.ru}[link:examples/one_domain.ru]
+# example for a standalone app.
+class Regurgitator::OneDomain
+  include Regurgitator::Endpoint
+
+  def initialize(app, opts) # :nodoc:
+    @domain = opts[:domain]
+    super(app, opts[:db])
+  end
+
+  def call(env) # :nodoc:
+    case env['REQUEST_METHOD']
+    when 'GET', 'HEAD'
+      serve_file(env, @domain, env['PATH_INFO'][1..-1])
+    else
+      @app.call(env)
+    end
+  end
+end

data/lib/regurgitator/server_settings.rb

@@ -0,0 +1,55 @@
+# -*- encoding: binary -*-
+# helpers for server_settings lookups
+module Regurgitator::ServerSettings
+
+  # :stopdoc:
+  SETTINGS_LOOKUP = 'SELECT value FROM server_settings WHERE field = ? LIMIT 1'
+  # :startdoc:
+
+  def initialize(*args) # :nodoc:
+    super
+    init_zone_ivars
+  end
+
+  def init_zone_ivars
+    @zone_cache_mtime = Time.at(0)
+    @zone_cache = nil
+    @zone_cache_lock = Mutex.new
+  end
+
+  def self.extended(obj)
+    obj.init_zone_ivars
+  end
+
+  def refresh_zone(force = false) # :nodoc:
+    @zone_cache_lock.synchronize { refresh_zone_unlocked(force) }
+  end
+
+  def refresh_zone_unlocked(force)
+    return @zone_cache if ! force && ((Time.now - @zone_cache_mtime) < 60)
+    tmp = Patricia.new
+    begin
+      row = @db[SETTINGS_LOOKUP, 'network_zones'].first or return tmp
+      row[:value].split(/\s*,\s*/).each do |zone|
+        row = @db[SETTINGS_LOOKUP, "zone_#{zone}"].first or next
+        begin
+          tmp.add(row[:value], zone)
+        rescue ArgumentError
+        end
+      end
+    ensure
+      @zone_cache_mtime = Time.now
+      return @zone_cache = tmp
+    end
+  end
+
+  # If +skip_local+ is true, it may return +:local+ if +addr+ is the address
+  # of the host this method is running on
+  # Returns the zone (a String) for a given +addr+, if there is one
+  # Returns +nil+ if zone information is unknown.
+  def zone_for(addr, skip_local = false)
+    return :local if skip_local && Regurgitator::Local.include?(addr)
+    zone = refresh_zone.search_best(addr) and return zone.data # String
+    nil
+  end
+end

data/pkg.mk

@@ -0,0 +1,175 @@
+RUBY = ruby
+RAKE = rake
+RSYNC = rsync
+WRONGDOC = wrongdoc
+
+GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
+	@./GIT-VERSION-GEN
+-include GIT-VERSION-FILE
+-include local.mk
+DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts RbConfig::CONFIG["DLEXT"]')
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+lib := lib
+
+ifeq ($(shell test -f script/isolate_for_tests && echo t),t)
+isolate_libs := tmp/isolate/$(RUBY_ENGINE)-$(RUBY_VERSION)/isolate.mk
+$(isolate_libs): script/isolate_for_tests
+	@$(RUBY) script/isolate_for_tests
+-include $(isolate_libs)
+lib := $(lib):$(ISOLATE_LIBS)
+endif
+
+ext := $(firstword $(wildcard ext/*))
+ifneq ($(ext),)
+ext_pfx := tmp/ext/$(RUBY_ENGINE)-$(RUBY_VERSION)
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(wildcard $(ext)/*.c $(ext_h))
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_d := $(ext_pfx)/$(ext)/.d
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>> $@
+$(ext_d):
+	@mkdir -p $(@D)
+	@> $@
+$(ext_pfx)/$(ext)/%: $(ext)/% $(ext_d)
+	install -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_d) $(ext_h)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	@echo $^ == $@
+	$(MAKE) -C $(@D)
+lib := $(lib):$(ext_pfx)/$(ext)
+build: $(ext_dl)
+else
+build:
+endif
+
+pkg_extra += GIT-VERSION-FILE NEWS ChangeLog LATEST
+ChangeLog: GIT-VERSION-FILE .wrongdoc.yml
+	$(WRONGDOC) prepare
+NEWS LATEST: ChangeLog
+
+manifest:
+	$(RM) .manifest
+	$(MAKE) .manifest
+
+.manifest: $(pkg_extra)
+	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
+		LC_ALL=C sort > $@+
+	cmp $@+ $@ || mv $@+ $@
+	$(RM) $@+
+
+doc:: .document .wrongdoc.yml $(pkg_extra)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(WRONGDOC) all
+	install -m644 COPYING doc/COPYING
+	install -m644 $(shell grep '^[A-Z]' .document) doc/
+
+ifneq ($(VERSION),)
+pkggem := pkg/$(rfpackage)-$(VERSION).gem
+pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
+release_notes := release_notes-$(VERSION)
+release_changes := release_changes-$(VERSION)
+
+release-notes: $(release_notes)
+release-changes: $(release_changes)
+$(release_changes):
+	$(WRONGDOC) release_changes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+$(release_notes):
+	$(WRONGDOC) release_notes > $@+
+	$(VISUAL) $@+ && test -s $@+ && mv $@+ $@
+
+# ensures we're actually on the tagged $(VERSION), only used for release
+verify:
+	test x"$(shell umask)" = x0022
+	git rev-parse --verify refs/tags/v$(VERSION)^{}
+	git diff-index --quiet HEAD^0
+	test $$(git rev-parse --verify HEAD^0) = \
+	     $$(git rev-parse --verify refs/tags/v$(VERSION)^{})
+
+fix-perms:
+	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
+	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
+
+gem: $(pkggem)
+
+install-gem: $(pkggem)
+	gem install $(CURDIR)/$<
+
+$(pkggem): manifest fix-perms
+	gem build $(rfpackage).gemspec
+	mkdir -p pkg
+	mv $(@F) $@
+
+$(pkgtgz): distdir = $(basename $@)
+$(pkgtgz): HEAD = v$(VERSION)
+$(pkgtgz): manifest fix-perms
+	@test -n "$(distdir)"
+	$(RM) -r $(distdir)
+	mkdir -p $(distdir)
+	tar cf - $$(cat .manifest) | (cd $(distdir) && tar xf -)
+	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
+	mv $@+ $@
+
+package: $(pkgtgz) $(pkggem)
+
+test-release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	@echo rubyforge add_release -f \
+	  -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	@echo gem push $(pkggem)
+	@echo rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+release:: verify package $(release_notes) $(release_changes)
+	# make tgz release on RubyForge
+	rubyforge add_release -f -n $(release_notes) -a $(release_changes) \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkgtgz)
+	# push gem to RubyGems.org
+	gem push $(pkggem)
+	# in case of gem downloads from RubyForge releases page
+	rubyforge add_file \
+	  $(rfproject) $(rfpackage) $(VERSION) $(pkggem)
+else
+gem install-gem: GIT-VERSION-FILE
+	$(MAKE) $@ VERSION=$(GIT_VERSION)
+endif
+
+all:: test
+test_units := $(wildcard test/test_*.rb)
+test: test-unit
+test-unit: $(test_units)
+$(test_units): build
+	$(RUBY) -I $(lib) $@ $(RUBY_TEST_OPTS)
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	find doc/images -type f | \
+		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+check-warnings:
+	@(for i in $$(git ls-files '*.rb'|grep -v '^setup\.rb$$'); \
+	  do $(RUBY) -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
+
+.PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest
+.PHONY: check-warnings