mogilefs-client 3.0.0 → 3.1.0

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 3
   prerelease: 
   segments: 
   - 3
+  - 1
   - 0
-  - 0
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors: 
 - Eric Wong
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-28 00:00:00 Z
+date: 2011-12-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -43,8 +43,8 @@ executables:
 - mog
 extensions: []
 
-extra_rdoc_files: 
-- Manifest.txt
+extra_rdoc_files: []
+
 files: 
 - .document
 - .gitignore
@@ -54,12 +54,11 @@ files:
 - HACKING
 - History
 - LICENSE
-- Manifest.txt
 - README
 - Rakefile
 - TODO
 - bin/mog
-- examples/mogstored_rack.rb
+- examples/stale_fid_checker.rb
 - lib/mogilefs.rb
 - lib/mogilefs/admin.rb
 - lib/mogilefs/backend.rb
@@ -72,6 +71,12 @@ files:
 - lib/mogilefs/http_reader.rb
 - lib/mogilefs/mogilefs.rb
 - lib/mogilefs/mysql.rb
+- lib/mogilefs/new_file.rb
+- lib/mogilefs/new_file/common.rb
+- lib/mogilefs/new_file/content_range.rb
+- lib/mogilefs/new_file/stream.rb
+- lib/mogilefs/new_file/tempfile.rb
+- lib/mogilefs/new_file/writer.rb
 - lib/mogilefs/paths_size.rb
 - lib/mogilefs/pool.rb
 - lib/mogilefs/socket.rb
@@ -104,7 +109,6 @@ files:
 - test/test_mogtool_bigfile.rb
 - test/test_mysql.rb
 - test/test_pool.rb
-- test/test_unit_mogstored_rack.rb
 - lib/mogilefs/version.rb
 - .gemtest
 homepage: http://bogomips.org/mogilefs-client
@@ -143,7 +147,6 @@ specification_version: 3
 summary: MogileFS client library for Ruby
 test_files: 
 - test/test_http_reader.rb
-- test/test_unit_mogstored_rack.rb
 - test/test_mogilefs_integration_list_keys.rb
 - test/test_db_backend.rb
 - test/test_mogilefs_socket_pure.rb

data/Manifest.txt

@@ -1,60 +0,0 @@
-.document
-.gitignore
-.wrongdoc.yml
-GIT-VERSION-GEN
-GNUmakefile
-HACKING
-History
-LICENSE
-Manifest.txt
-README
-Rakefile
-TODO
-bin/mog
-examples/mogstored_rack.rb
-lib/mogilefs.rb
-lib/mogilefs/admin.rb
-lib/mogilefs/backend.rb
-lib/mogilefs/bigfile.rb
-lib/mogilefs/bigfile/filter.rb
-lib/mogilefs/chunker.rb
-lib/mogilefs/client.rb
-lib/mogilefs/copy_stream.rb
-lib/mogilefs/http_file.rb
-lib/mogilefs/http_reader.rb
-lib/mogilefs/mogilefs.rb
-lib/mogilefs/mysql.rb
-lib/mogilefs/paths_size.rb
-lib/mogilefs/pool.rb
-lib/mogilefs/socket.rb
-lib/mogilefs/socket/kgio.rb
-lib/mogilefs/socket/pure_ruby.rb
-lib/mogilefs/socket_common.rb
-lib/mogilefs/util.rb
-setup.rb
-test/.gitignore
-test/aggregate.rb
-test/exec.rb
-test/fresh.rb
-test/integration.rb
-test/setup.rb
-test/socket_test.rb
-test/test_admin.rb
-test/test_backend.rb
-test/test_bigfile.rb
-test/test_client.rb
-test/test_db_backend.rb
-test/test_fresh.rb
-test/test_http_reader.rb
-test/test_mogilefs.rb
-test/test_mogilefs_integration.rb
-test/test_mogilefs_integration_large_pipe.rb
-test/test_mogilefs_integration_list_keys.rb
-test/test_mogilefs_socket_kgio.rb
-test/test_mogilefs_socket_pure.rb
-test/test_mogstored_rack.rb
-test/test_mogtool_bigfile.rb
-test/test_mysql.rb
-test/test_pool.rb
-test/test_unit_mogstored_rack.rb
-lib/mogilefs/version.rb

data/examples/mogstored_rack.rb

@@ -1,188 +0,0 @@
-# -*- encoding: binary -*-
-require 'digest/md5'
-require 'rack'
-
-# Rack application for handling HTTP PUT/DELETE/MKCOL operations needed
-# for a MogileFS storage server.  GET requests are handled by
-# Rack::File and Rack::Head _must_ be in the middleware stack for
-# mogilefsd fsck to work properly with keepalive.
-#
-# Usage in rackup config file (config.ru):
-#
-#    require "./mogstored_rack"
-#    use Rack::Head
-#    run MogstoredRack.new("/var/mogdata")
-class MogstoredRack
-  class ContentMD5 < Digest::MD5
-    def content_md5
-      [ digest ].pack("m").strip!
-    end
-  end
-
-  def initialize(root, opts = {})
-    @root = File.expand_path(root)
-    @io_size = opts[:io_size] || 0x100000
-    @rack_file = (opts[:app] || Rack::File.new(@root))
-    @fsync = !! opts[:fsync]
-    @creat_perms = opts[:creat_perms] || (~File.umask & 0666)
-    @mkdir_perms = opts[:mkdir_perms] || (~File.umask & 0777)
-    @reread_verify = !! opts[:reread_verify]
-    @open_flags = opts[:open_flags] || 0
-    @open_flags |= IO::RDWR | IO::CREAT | IO::EXCL
-  end
-
-  def tmpfile(basename, dir)
-    t = Time.now.utc.strftime("%Y%m%d%H%M%S")
-    seq = 0
-    begin
-      fp = File.open("#{dir}/#{basename}.#{t}.#{seq}.tmp", @open_flags, 0600)
-    rescue Errno::EEXIST
-      retry if (seq += 1) < 10
-      raise
-    end
-    fp.binmode
-    fp.sync = true
-    fp
-  end
-
-  def call(env)
-    case env["REQUEST_METHOD"]
-    when "GET", "HEAD"
-      case env["PATH_INFO"]
-      when "/"
-        r(200, "") # MogileFS seems to need this...
-      else
-        @rack_file.call(env)
-      end
-    when "PUT"
-      put(env)
-    when "DELETE"
-      delete(env)
-    when "MKCOL"
-      mkcol(env)
-    else
-      r(405, "unsupported method", env)
-    end
-    rescue Errno::EPERM, Errno::EACCES => err
-      r(403, "#{err.message} (#{err.class})", env)
-    rescue => err
-      r(500, "#{err.message} (#{err.class})", env)
-  end
-
-  def mkcol(env)
-    path = server_path(env) or return r(400)
-    Dir.mkdir(path, @mkdir_perms)
-    r(204)
-    rescue Errno::EEXIST # succeed (204) on race condition
-      File.directory?(path) ? r(204) : r(409)
-  end
-
-  def delete(env)
-    path = server_path(env) or return r(400)
-    File.exist?(path) or return r(404)
-    File.directory?(path) ? Dir.rmdir(path) : File.unlink(path)
-    r(204)
-    rescue Errno::ENOENT # return 404 on race condition
-      File.exist?(path) ? r(500) : r(404)
-  end
-
-  def put(env)
-    path = server_path(env) or return r(400)
-    dir = File.dirname(path)
-    File.directory?(dir) or return r(403)
-
-    tmp = tmpfile(File.basename(path), dir)
-    buf = ""
-    received = put_loop(env["rack.input"], tmp, buf)
-    err = content_md5_fail?(env, received) and return err
-    if @reread_verify && err = reread_md5_fail?(env, tmp, received, buf)
-      return err
-    end
-    tmp.chmod(@creat_perms)
-    File.rename(tmp.path, path)
-    fsync(dir, tmp) if @fsync
-    resp = r(201)
-    resp[1]["X-Received-Content-MD5"] = received
-    resp
-    rescue
-      File.unlink(tmp.path) if tmp && File.exist?(tmp.path)
-      raise
-    ensure
-      tmp.close if tmp
-  end
-
-  def put_loop(src, dst, buf)
-    md5 = ContentMD5.new
-    while src.read(@io_size, buf)
-      md5.update(buf)
-      dst.write(buf)
-    end
-    md5.content_md5
-  end
-
-  def server_path(env)
-    path = env['PATH_INFO'].squeeze('/')
-    path.split(%r{/}).include?("..") and return false
-    "#@root#{path}"
-  end
-
-  # returns a plain-text HTTP response
-  def r(code, msg = nil, env = nil)
-    if env && logger = env["rack.logger"]
-      logger.warn("#{env['REQUEST_METHOD']} #{env['PATH_INFO']} " \
-                  "#{code} #{msg.inspect}")
-    end
-    if Rack::Utils::STATUS_WITH_NO_ENTITY_BODY.include?(code)
-      [ code, {}, [] ]
-    else
-      msg ||= Rack::Utils::HTTP_STATUS_CODES[code] || ""
-      msg += "\n" if msg.size > 0
-      [ code,
-        { 'Content-Type' => 'text/plain', 'Content-Length' => msg.size.to_s },
-        [ msg ] ]
-    end
-  end
-
-  # Tries to detect filesystem/disk corruption.
-  # Unfortunately, posix_fadvise(2)/IO#advise is only advisory and
-  # can't guarantee we're not just reading the data in the kernel
-  # page cache.
-  def reread_md5_fail?(env, tmp, received, buf)
-    # try to force a reread from the storage device, not cache
-    tmp.fsync
-    tmp.rewind
-    tmp.advise(:dontneed) rescue nil # only in Ruby 1.9.3 and only advisory
-
-    md5 = ContentMD5.new
-    while tmp.read(0x4000, buf)
-      md5.update(buf)
-    end
-    reread = md5.content_md5
-    reread == received and return false # success
-    r(500, "reread MD5 mismatch\n" \
-           "received: #{received}\n" \
-           "  reread: #{reread}", env)
-  end
-
-  # Tries to detect network corruption by verifying the client-supplied
-  # Content-MD5 is correct.  It's highly unlikely the MD5 can be corrupted
-  # in a way that also allows corrupt data to pass through.
-  #
-  # The Rainbows!/Unicorn HTTP servers will populate the HTTP_CONTENT_MD5
-  # field in +env+ after env["rack.input"] is fully-consumed.  Clients
-  # may also send Content-MD5 as a header and this will still work.
-  def content_md5_fail?(env, received)
-    expected = env["HTTP_CONTENT_MD5"] or return false
-    expected = expected.strip
-    expected == received and return false # success
-    r(400, "Content-MD5 mismatch\n" \
-           "expected: #{expected}\n" \
-           "received: #{received}", env)
-  end
-
-  # fsync each and every directory component above us on the same device
-  def fsync(dir, tmp)
-    tmp.fsync
-    File.open(dir) { |io| io.fsync }
-  end
-end

data/test/test_unit_mogstored_rack.rb

@@ -1,72 +0,0 @@
-# -*- encoding: binary -*-
-require "test/unit"
-require "tmpdir"
-require "fileutils"
-begin
-  require "./examples/mogstored_rack"
-rescue LoadError
-end
-
-class TestUnitMogstoredRack < Test::Unit::TestCase
-  attr_reader :req
-
-  def setup
-    @docroot = Dir.mktmpdir(["mogstored_rack", ".docroot"])
-  end
-
-  def test_defaults
-    req = Rack::MockRequest.new(MogstoredRack.new(@docroot))
-    all_methods(req)
-  end
-
-  def test_fsync_true
-    req = Rack::MockRequest.new(MogstoredRack.new(@docroot, :fsync=>true))
-    all_methods(req)
-  end
-
-  def test_reread_verify
-    app = MogstoredRack.new(@docroot, :reread_verify=>true)
-    req = Rack::MockRequest.new(app)
-    all_methods(req)
-  end
-
-  def all_methods(req)
-    assert_equal 200, req.get("/").status
-    assert ! File.directory?("#@docroot/dev666")
-    assert_equal 204, req.request("MKCOL", "/dev666").status
-    assert File.directory?("#@docroot/dev666")
-
-    io = StringIO.new("HELLO")
-    r = req.request("PUT", "/dev666/666.fid", :input => io)
-    assert_equal 201, r.status
-    assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
-
-    # invalid MD5
-    io = StringIO.new("WORLD")
-    md5 = [ Digest::MD5.new.digest ].pack("m").strip!
-    opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
-    r = req.request("PUT", "/dev666/666.fid", opts)
-    assert_equal 400, r.status
-    assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
-
-    # valid MD5
-    io = StringIO.new("VALID")
-    md5 = [ Digest::MD5.digest("VALID") ].pack("m").strip!
-    opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
-    r = req.request("PUT", "/dev666/666.fid", opts)
-    assert_equal 201, r.status
-    assert_equal "VALID", IO.read("#@docroot/dev666/666.fid")
-
-    r = req.request("GET", "/dev666/666.fid")
-    assert_equal 200, r.status
-    assert_equal "VALID", r.body
-
-    r = req.request("DELETE", "/dev666/666.fid")
-    assert_equal 204, r.status
-    assert ! File.exist?("#@docroot/dev666/666.fid")
-  end
-
-  def teardown
-    FileUtils.rmtree(@docroot)
-  end
-end if defined?(Rack)