regurgitator 0.1.0 → 0.2.0

data/.manifest

@@ -22,6 +22,7 @@ lib/regurgitator/domain_path.rb
 lib/regurgitator/endpoint.rb
 lib/regurgitator/file_info.rb
 lib/regurgitator/file_request.rb
+lib/regurgitator/list_keys.rb
 lib/regurgitator/local.rb
 lib/regurgitator/local_file.rb
 lib/regurgitator/one_domain.rb
@@ -42,5 +43,6 @@ t/t0001-domain_path.sh
 t/t0002-domain_host.sh
 t/t0003-one_domain.sh
 t/test-lib.sh
+test/test_list_keys.rb
 test/test_local.rb
 test/test_server_settings.rb

data/ChangeLog

@@ -1,5 +1,68 @@
 ChangeLog from http://bogomips.org/regurgitator.git
 
+   commit 4874e7d38453a0b4b3f47d8dc8095a314083bd11
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Dec 15 18:11:02 2011 -0800
+   
+       regurgitator 0.2.0
+       
+       * add list_keys module for listing keys
+       * add :reproxy_key, overriding global REPROXY_KEY constant
+       * remove support for altip/altmask (hardly ever used)
+   
+   commit ed9f6b55d3acbe8bf0ee8599fff466fbe9eabd5d
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Dec 15 18:16:19 2011 -0800
+   
+       strip out the altmask/altip handling stuff
+       
+       Hardly anyone ever uses/used it, so stop attempting to support
+       it to remove the risk of supporting it improperly.
+   
+   commit 3052e1b0f75523b46f43ddb1163b37c12e6ed57e
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Dec 1 00:13:43 2011 -0800
+   
+       list_keys: force case-sensitive key name matching for MySQL
+       
+       Case-insensitivity kills millions each year...
+       
+       We require Sequel 3.29+ for SQLite, so we avoid this problem
+       there already.
+   
+   commit 045b555d7404ca5bfac5cf69a5e7ab35872602ef
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Wed Nov 16 19:25:32 2011 -0800
+   
+       add :reproxy_key, overriding global REPROXY_KEY constant
+       
+       Users may never want reproxy behavior, or they may want to
+       use a different key, so support both use cases.
+       
+       Reproxy support (for nginx) is now off now by default.
+   
+   commit 80982a28eafda2b439bbb1545e649509f29e83d4
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Wed Nov 16 19:24:59 2011 -0800
+   
+       t: update my-tap-lib.sh
+       
+       Fix race conditions in verbose mode
+   
+   commit 530ab82be1608f6a136169aee71dc5745b9b1785
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Wed Nov 16 18:58:47 2011 -0800
+   
+       add list_keys module for listing keys
+       
+       This may be useful for building directory indices.
+   
+   commit 611cb7223064f99126388062c8197d4271859b92
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Fri Nov 11 08:49:49 2011 +0000
+   
+       pkg.mk: update to the latest version
+   
    commit 84e48c63074049f32660672b5a3c195d12dc3896
    Author: Eric Wong <normalperson@yhbt.net>
    Date:   Fri Nov 11 08:46:12 2011 +0000

data/GIT-VERSION-FILE

@@ -1 +1 @@
-GIT_VERSION = 0.1.0
+GIT_VERSION = 0.2.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v0.1.0.GIT
+DEF_VER=v0.2.0.GIT
 
 LF='
 '

data/LATEST

@@ -1,6 +1,6 @@
-=== regurgitator 0.1.0 / 2011-11-11 08:47 UTC
+=== regurgitator 0.2.0 / 2011-12-16 02:27 UTC
 
-  * local_file: support HTTP Range: requests
-  * minor code cleanups
-  * update/fix install dependencies (include rpatricia)
+  * add list_keys module for listing keys
+  * add :reproxy_key, overriding global REPROXY_KEY constant
+  * remove support for altip/altmask (hardly ever used)
 

data/NEWS

@@ -1,3 +1,9 @@
+=== regurgitator 0.2.0 / 2011-12-16 02:27 UTC
+
+  * add list_keys module for listing keys
+  * add :reproxy_key, overriding global REPROXY_KEY constant
+  * remove support for altip/altmask (hardly ever used)
+
 === regurgitator 0.1.0 / 2011-11-11 08:47 UTC
 
   * local_file: support HTTP Range: requests

data/lib/regurgitator.rb

@@ -13,10 +13,6 @@ require 'rpatricia'
 # so just "require 'regurgitator'" in your code.
 module Regurgitator
 
-  # This HTTP header is read to determine a location to reproxy to.
-  # Add this header in your nginx config.
-  REPROXY_KEY = 'HTTP_X_REPROXY_PATH'
-
   autoload :ServerSettings, 'regurgitator/server_settings'
   autoload :Domain, 'regurgitator/domain'
   autoload :Device, 'regurgitator/device'
@@ -25,7 +21,7 @@ module Regurgitator
   autoload :FileRequest, 'regurgitator/file_request'
   autoload :Local, 'regurgitator/local'
   autoload :LocalFile, 'regurgitator/local_file'
-
+  autoload :ListKeys, 'regurgitator/list_keys'
   # Rack middlewares/apps
   autoload :DomainPath, 'regurgitator/domain_path'
   autoload :DomainHost, 'regurgitator/domain_host'

data/lib/regurgitator/device.rb

@@ -5,7 +5,7 @@ module Regurgitator::Device
 
   # :stopdoc:
   REFRESH_DEVICE = <<-EOS
-    SELECT d.devid, h.hostip, h.altip, h.altmask, h.http_port, h.http_get_port
+    SELECT d.devid, h.hostip, h.http_port, h.http_get_port
     FROM device d
       LEFT JOIN host h ON d.hostid = h.hostid
     WHERE d.status IN ('readonly','alive','drain') AND h.status = 'alive'
@@ -54,18 +54,7 @@ module Regurgitator::Device
       x[:zone] = zone_for(hostip)
       devid = x[:devid]
       o = { :path => "/dev#{devid}", :port => port, :host => hostip }
-
       x[:uris] = { :pri => device_uri_pair!(o, get_port) }
-      x[:ipaddr][:altmask] = pat = Patricia.new
-      altmask = x[:altmask] and pat.add(altmask)
-
-      if altip = x[:altip]
-        x[:ipaddr][:altip] = altip
-        o[:host] = altip
-        o[:port] = port
-        x[:uris][:alt] = device_uri_pair!(o, get_port)
-      end
-
       tmp[devid] = x
     end
     Regurgitator::Local.refresh_addrs!

data/lib/regurgitator/domain_host.rb

@@ -27,7 +27,7 @@ class Regurgitator::DomainHost
       raise TypeError, ":suffix must be a String or Regexp #{suffix.inspect}"
     end
     @domain_regexp = suffix
-    endpoint_init(app, opts[:db])
+    endpoint_init(app, opts[:db], opts[:reproxy_key])
   end
 
   def call(env) # :nodoc:

data/lib/regurgitator/domain_path.rb

@@ -27,7 +27,8 @@ class Regurgitator::DomainPath
     end
   end
 
-  def initialize(app, db) # :nodoc:
-    endpoint_init(app, Hash === db ? db[:db] : db)
+  def initialize(app, opts) # :nodoc:
+    opts = { :db => opts } unless Hash === opts
+    endpoint_init(app, opts[:db], opts[:reproxy_key])
   end
 end

data/lib/regurgitator/endpoint.rb

@@ -5,21 +5,20 @@ module Regurgitator::Endpoint
   include Rack::Utils
   include Rack::Mime
 
-  REPROXY_KEY = Regurgitator::REPROXY_KEY # :nodoc:
-
-  def endpoint_init(app, db)
+  def endpoint_init(app, db, reproxy_key = nil)
     @app = app
     @db = db
+    @reproxy_key = reproxy_key
     file_info_init
   end
 
   def best_addr(env)
-    env.include?(REPROXY_KEY) ? env['REMOTE_ADDR'] : Regurgitator::Local.addr
+    env.include?(@reproxy_key) ? env['REMOTE_ADDR'] : Regurgitator::Local.addr
   end
 
   # see nginx config examples for reproxy
   def reproxy_path(env)
-    env[REPROXY_KEY]
+    env[@reproxy_key]
   end
 
   # allow users to specify desired "Save As" filenames in the query string

data/lib/regurgitator/file_info.rb

@@ -37,7 +37,7 @@ module Regurgitator::FileInfo
       devinfo = devices[x[:devid]] or next
       fid >= 10_000_000_000 and next # TODO: support larger FIDs
 
-      uris = devinfo[:uris][alt_zone?(zone, devinfo) ? :alt : :pri]
+      uris = devinfo[:uris][:pri]
       nfid = sprintf('%010u', fid)
       /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
       fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
@@ -51,8 +51,4 @@ module Regurgitator::FileInfo
     uris = info[:uris] = read_uris.empty? ? drain_uris : read_uris
     (uris.empty? && ! update) ? file_info(env, domain, dkey, true) : info
   end
-
-  def alt_zone?(zone, devinfo)
-    zone == :alt || devinfo[:ipaddr][:altmask].include?(zone)
-  end
 end

data/lib/regurgitator/list_keys.rb

@@ -0,0 +1,27 @@
+# -*- encoding: binary -*-
+module Regurgitator::ListKeys
+  include Regurgitator::Domain
+
+  LIST_KEYS_MAX = 1000
+
+  def list_keys(domain, opts = {})
+    dmid = get_dmid(domain) or return
+
+    after, limit, prefix = opts[:after], opts[:limit], opts[:prefix]
+    limit ||= LIST_KEYS_MAX
+
+    Integer === limit or
+      raise ArgumentError, ":limit not an Integer=#{limit.inspect}"
+
+    limit = LIST_KEYS_MAX if limit > LIST_KEYS_MAX
+    sql = "SELECT fid,dkey,length,devcount,classid " \
+          "FROM file WHERE dmid = #{dmid}"
+    sql << " AND dkey LIKE /*! BINARY */ #{@db.literal(prefix+'%')}" if prefix
+    sql << " AND dkey > #{@db.literal(after)}" if after
+    sql << " ORDER BY dkey LIMIT #{limit}"
+
+    # we don't enforce collation here, give DBAs some freedom...
+    ds = @db[sql]
+    block_given? ? ds.each { |row| yield(row) } : ds.to_a
+  end
+end

data/lib/regurgitator/one_domain.rb

@@ -19,7 +19,7 @@ class Regurgitator::OneDomain
 
   def initialize(app, opts) # :nodoc:
     @domain = opts[:domain]
-    endpoint_init(app, opts[:db])
+    endpoint_init(app, opts[:db], opts[:reproxy_key])
   end
 
   def call(env) # :nodoc:

data/pkg.mk

@@ -69,7 +69,7 @@ doc:: .document .wrongdoc.yml $(pkg_extra)
 	$(RM) -r doc
 	$(WRONGDOC) all
 	install -m644 COPYING doc/COPYING
-	install -m644 $(shell grep '^[A-Z]' .document) doc/
+	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
 
 ifneq ($(VERSION),)
 pkggem := pkg/$(rfpackage)-$(VERSION).gem
@@ -168,7 +168,7 @@ doc_gz:
 	for i in $(docs); do \
 	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
 check-warnings:
-	@(for i in $$(git ls-files '*.rb'|grep -v '^setup\.rb$$'); \
+	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
 	  do $(RUBY) -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
 
 .PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest

data/t/domain_host.ru

@@ -4,5 +4,6 @@ use Rack::ContentType, 'text/plain'
 o = {
   :db => Sequel.connect(ENV['TEST_SEQUEL']),
   :suffix => '.example.com',
+  :reproxy_key => "HTTP_X_REPROXY_PATH",
 }
 run Regurgitator::DomainHost.new(lambda { |env| [ 404, [], [] ] }, o)

data/t/domain_path.ru

@@ -1,5 +1,8 @@
 require 'regurgitator'
 use Rack::ContentLength
 use Rack::ContentType, 'text/plain'
-db = Sequel.connect(ENV['TEST_SEQUEL'])
-run Regurgitator::DomainPath.new(lambda { |env| [ 404, [], [] ] }, db)
+o = {
+  :db => Sequel.connect(ENV['TEST_SEQUEL']),
+  :reproxy_key => "HTTP_X_REPROXY_PATH",
+}
+run Regurgitator::DomainPath.new(lambda { |env| [ 404, [], [] ] }, o)

data/t/my-tap-lib.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (c) 2009 Eric Wong <normalperson@yhbt.net>
+# Copyright (c) 2009, 2010 Eric Wong <normalperson@yhbt.net>
 #
 # TAP-producing shell library for POSIX-compliant Bourne shells We do
 # not _rely_ on Bourne Again features, though we will use "set -o
@@ -188,9 +188,10 @@ then
 
 	(
 		# use a subshell so seds are not waitable
-		$SED -e 's/^/#: /' $t_stdout &
-		$SED -e 's/^/#! /' $t_stderr &
+		$SED -e 's/^/#: /' < $t_stdout &
+		$SED -e 's/^/#! /' < $t_stderr &
 	) &
+	wait
 	exec > $t_stdout 2> $t_stderr
 else
 	exec > /dev/null 2> /dev/null

data/t/one_domain.ru

@@ -4,5 +4,6 @@ use Rack::ContentType, 'text/plain'
 o = {
   :db => Sequel.connect(ENV['TEST_SEQUEL']),
   :domain => 'd',
+  :reproxy_key => "HTTP_X_REPROXY_PATH",
 }
 run Regurgitator::OneDomain.new(lambda { |env| [ 404, [], [] ] }, o)

data/test/test_list_keys.rb

@@ -0,0 +1,40 @@
+require "test/unit"
+require "regurgitator"
+require "tempfile"
+
+class TestListKeys < Test::Unit::TestCase
+  include Regurgitator::ListKeys
+  def setup
+    @tmp = Tempfile.new(%w(regurgitator_list_keys .sqlite3))
+    rv = system("sqlite3 #{@tmp.path} < t/fixtures.sql")
+    assert rv, $?.inspect
+    @db = Sequel.connect("sqlite://#{@tmp.path}")
+    domain_init
+  end
+
+  def test_list_keys
+    a = list_keys("d", {})
+    assert_kind_of Array, a
+    row = a[0]
+    assert_equal "blah", row[:dkey]
+    [ :fid, :length, :devcount, :classid ].each do |key|
+      assert_kind_of Integer, row[key]
+    end
+
+    list_keys("d", {:prefix => "bla"})
+    assert_equal row, a[0]
+    assert_empty list_keys("d", {:prefix => "blahh"})
+    assert_empty list_keys("d", {:prefix => "bla", :after => "blah"})
+
+    assert_nil list_keys("afadf")
+    list_keys("d") { |r| assert_equal row, r }
+  end
+
+  def test_list_keys_insensitive
+    a = list_keys("d", :prefix => "B")
+    assert a.empty?, a.inspect << " not empty"
+    a = list_keys("d", :prefix => "b")
+    assert_equal 1, a.size
+    assert_equal "blah", a[0][:dkey]
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: regurgitator
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Regurgitators
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-11 00:00:00 Z
+date: 2011-12-16 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rack
@@ -111,6 +111,7 @@ extra_rdoc_files:
 - lib/regurgitator/endpoint.rb
 - lib/regurgitator/file_info.rb
 - lib/regurgitator/file_request.rb
+- lib/regurgitator/list_keys.rb
 - lib/regurgitator/local.rb
 - lib/regurgitator/local_file.rb
 - lib/regurgitator/one_domain.rb
@@ -143,6 +144,7 @@ files:
 - lib/regurgitator/endpoint.rb
 - lib/regurgitator/file_info.rb
 - lib/regurgitator/file_request.rb
+- lib/regurgitator/list_keys.rb
 - lib/regurgitator/local.rb
 - lib/regurgitator/local_file.rb
 - lib/regurgitator/one_domain.rb
@@ -163,6 +165,7 @@ files:
 - t/t0002-domain_host.sh
 - t/t0003-one_domain.sh
 - t/test-lib.sh
+- test/test_list_keys.rb
 - test/test_local.rb
 - test/test_server_settings.rb
 homepage: http://bogomips.org/regurgitator/
@@ -197,10 +200,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: rainbows
-rubygems_version: 1.8.5
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: read-only Rack endpoints for MogileFS
 test_files: 
 - test/test_local.rb
 - test/test_server_settings.rb
+- test/test_list_keys.rb