mogilefs-client 2.2.0 → 3.0.0.rc1

data/test/test_mysql.rb

@@ -1,6 +1,5 @@
 # -*- encoding: binary -*-
-require 'test/setup'
-require 'mogilefs/mysql'
+require './test/setup'
 
 class MogileFS::Mysql
   public :refresh_device

data/test/test_pool.rb

@@ -2,7 +2,7 @@
 require 'test/unit'
 
 $TESTING = true
-
+require 'mogilefs'
 require 'mogilefs/pool'
 
 class MogileFS::Pool

data/test/test_unit_mogstored_rack.rb

@@ -0,0 +1,72 @@
+# -*- encoding: binary -*-
+require "test/unit"
+require "tmpdir"
+require "fileutils"
+begin
+  require "./examples/mogstored_rack"
+rescue LoadError
+end
+
+class TestUnitMogstoredRack < Test::Unit::TestCase
+  attr_reader :req
+
+  def setup
+    @docroot = Dir.mktmpdir(["mogstored_rack", ".docroot"])
+  end
+
+  def test_defaults
+    req = Rack::MockRequest.new(MogstoredRack.new(@docroot))
+    all_methods(req)
+  end
+
+  def test_fsync_true
+    req = Rack::MockRequest.new(MogstoredRack.new(@docroot, :fsync=>true))
+    all_methods(req)
+  end
+
+  def test_reread_verify
+    app = MogstoredRack.new(@docroot, :reread_verify=>true)
+    req = Rack::MockRequest.new(app)
+    all_methods(req)
+  end
+
+  def all_methods(req)
+    assert_equal 200, req.get("/").status
+    assert ! File.directory?("#@docroot/dev666")
+    assert_equal 204, req.request("MKCOL", "/dev666").status
+    assert File.directory?("#@docroot/dev666")
+
+    io = StringIO.new("HELLO")
+    r = req.request("PUT", "/dev666/666.fid", :input => io)
+    assert_equal 201, r.status
+    assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
+
+    # invalid MD5
+    io = StringIO.new("WORLD")
+    md5 = [ Digest::MD5.new.digest ].pack("m").strip!
+    opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
+    r = req.request("PUT", "/dev666/666.fid", opts)
+    assert_equal 400, r.status
+    assert_equal "HELLO", IO.read("#@docroot/dev666/666.fid")
+
+    # valid MD5
+    io = StringIO.new("VALID")
+    md5 = [ Digest::MD5.digest("VALID") ].pack("m").strip!
+    opts = { :input => io, "HTTP_CONTENT_MD5" => md5 }
+    r = req.request("PUT", "/dev666/666.fid", opts)
+    assert_equal 201, r.status
+    assert_equal "VALID", IO.read("#@docroot/dev666/666.fid")
+
+    r = req.request("GET", "/dev666/666.fid")
+    assert_equal 200, r.status
+    assert_equal "VALID", r.body
+
+    r = req.request("DELETE", "/dev666/666.fid")
+    assert_equal 204, r.status
+    assert ! File.exist?("#@docroot/dev666/666.fid")
+  end
+
+  def teardown
+    FileUtils.rmtree(@docroot)
+  end
+end if defined?(Rack)

metadata

@@ -1,109 +1,121 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  hash: 7
-  prerelease: false
+  hash: 15424119
+  prerelease: 6
   segments: 
-  - 2
-  - 2
+  - 3
+  - 0
   - 0
-  version: 2.2.0
+  - rc
+  - 1
+  version: 3.0.0.rc1
 platform: ruby
 authors: 
 - Eric Wong
-- Eric Hodel
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2011-01-13 00:00:00 +00:00
-default_executable: 
+date: 2011-11-21 00:00:00 Z
 dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: ZenTest
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 3
-        - 6
-        - 1
-        version: 3.6.1
-  type: :development
-  version_requirements: *id001
 - !ruby/object:Gem::Dependency 
   name: hoe
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 47
+        hash: 27
         segments: 
         - 2
-        - 8
-        - 0
-        version: 2.8.0
+        - 12
+        version: "2.12"
   type: :development
-  version_requirements: *id002
-description: "Source repository (git):"
-email: normalperson@yhbt.net
+  version_requirements: *id001
+description: |-
+  A MogileFS client library for Ruby.  MogileFS is an open source
+  distributed filesystem, see: http://mogilefs.org for more details.  This
+  library allows any Ruby application to read, write and delete files in a
+  MogileFS instance.
+email: 
+- normalperson@yhbt.net
 executables: 
 - mog
 extensions: []
 
 extra_rdoc_files: 
-- History.txt
-- LICENSE.txt
 - Manifest.txt
-- README.txt
 files: 
+- .document
 - .gitignore
+- .wrongdoc.yml
+- GIT-VERSION-GEN
 - GNUmakefile
-- History.txt
-- LICENSE.txt
+- HACKING
+- History
+- LICENSE
 - Manifest.txt
-- README.txt
+- README
 - Rakefile
+- TODO
 - bin/mog
+- examples/mogstored_rack.rb
 - lib/mogilefs.rb
 - lib/mogilefs/admin.rb
 - lib/mogilefs/backend.rb
 - lib/mogilefs/bigfile.rb
+- lib/mogilefs/bigfile/filter.rb
+- lib/mogilefs/chunker.rb
 - lib/mogilefs/client.rb
-- lib/mogilefs/httpfile.rb
+- lib/mogilefs/copy_stream.rb
+- lib/mogilefs/http_file.rb
+- lib/mogilefs/http_reader.rb
 - lib/mogilefs/mogilefs.rb
 - lib/mogilefs/mysql.rb
-- lib/mogilefs/network.rb
+- lib/mogilefs/paths_size.rb
 - lib/mogilefs/pool.rb
+- lib/mogilefs/socket.rb
+- lib/mogilefs/socket/kgio.rb
+- lib/mogilefs/socket/pure_ruby.rb
+- lib/mogilefs/socket_common.rb
 - lib/mogilefs/util.rb
 - setup.rb
 - test/.gitignore
 - test/aggregate.rb
+- test/exec.rb
+- test/fresh.rb
+- test/integration.rb
 - test/setup.rb
+- test/socket_test.rb
 - test/test_admin.rb
 - test/test_backend.rb
 - test/test_bigfile.rb
 - test/test_client.rb
 - test/test_db_backend.rb
+- test/test_fresh.rb
+- test/test_http_reader.rb
 - test/test_mogilefs.rb
+- test/test_mogilefs_integration.rb
+- test/test_mogilefs_integration_large_pipe.rb
+- test/test_mogilefs_integration_list_keys.rb
+- test/test_mogilefs_socket_kgio.rb
+- test/test_mogilefs_socket_pure.rb
+- test/test_mogstored_rack.rb
+- test/test_mogtool_bigfile.rb
 - test/test_mysql.rb
-- test/test_network.rb
 - test/test_pool.rb
-- test/test_util.rb
-has_rdoc: true
-homepage: http://seattlerb.rubyforge.org/mogilefs-client
+- test/test_unit_mogstored_rack.rb
+- lib/mogilefs/version.rb
+- .gemtest
+homepage: http://bogomips.org/mogilefs-client
 licenses: []
 
 post_install_message: 
 rdoc_options: 
 - --main
-- README.txt
+- README
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -118,27 +130,37 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: seattlerb
-rubygems_version: 1.3.7
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
-summary: A Ruby MogileFS client
+summary: MogileFS client library for Ruby
 test_files: 
+- test/test_http_reader.rb
+- test/test_unit_mogstored_rack.rb
+- test/test_mogilefs_integration_list_keys.rb
 - test/test_db_backend.rb
-- test/test_network.rb
+- test/test_mogilefs_socket_pure.rb
 - test/test_mysql.rb
-- test/test_util.rb
 - test/test_backend.rb
 - test/test_mogilefs.rb
+- test/test_mogtool_bigfile.rb
 - test/test_client.rb
 - test/test_admin.rb
+- test/test_mogilefs_socket_kgio.rb
+- test/test_mogilefs_integration.rb
+- test/test_fresh.rb
 - test/test_pool.rb
 - test/test_bigfile.rb
+- test/test_mogilefs_integration_large_pipe.rb
+- test/test_mogstored_rack.rb

data/README.txt

@@ -1,80 +0,0 @@
-= mogilefs-client
-
-A Ruby MogileFS client
-
-Rubyforge Project:
-
-http://rubyforge.org/projects/seattlerb/
-
-Documentation:
-
-http://seattlerb.rubyforge.org/mogilefs-client
-
-File bugs:
-
-http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
-
-Please email Eric Wong at normalperson@yhbt.net as well since
-he finds web interfaces difficult to use.
-
-Source repository (git):
-
-  git://git.bogomips.org/mogilefs-client.git
-  http://git.bogomips.org/mogilefs-client.git
-  git://repo.or.cz/ruby-mogilefs-client.git (mirror)
-  http://repo.or.cz/r/ruby-mogilefs-client.git (mirror)
-
-Repository browsers:
-
-* http://git.bogomips.org/cgit/mogilefs-client.git (cgit)
-* http://repo.or.cz/w/ruby-mogilefs-client.git (gitweb mirror)
-
-== About
-
-A Ruby MogileFS client.  MogileFS is a distributed filesystem written
-by Danga Interactive.  This client only supports HTTP.
-
-For information on MogileFS see:
-
-http://danga.com/mogilefs/
-
-== Installing mogilefs-client
-
-First you need a MogileFS setup.  You can find information on how to do
-that at the above URL.
-
-Then install the gem:
-
-  $ sudo gem install mogilefs-client
-
-== Using mogilefs-client
-
-  # Create a new instance that will communicate with these trackers:
-  hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
-  mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts)
-
-  # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
-  mg.store_content 'some_key', 'text', "A bunch of text to store"
-
-  # Retrieve data from 'some_key'
-  data = mg.get_file_data 'some_key'
-
-  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
-  # 'image'.
-  mg.store_file 'my_image', 'image', 'image.jpeg'
-
-  # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
-  # 'image' using an open IO.
-  File.open 'image.jpeg', 'rb' do |fp|
-    mg.store_file 'my_image', 'image', fp
-  end
-
-  # Remove the key 'my_image' and 'some_key'.
-  mg.delete 'my_image'
-  mg.delete 'some_key'
-
-== WARNING!
-
-This client is only supported in HTTP mode.  NFS mode was previously
-supported in 1.3.x, but since MogileFS 2.x has dropped support for
-NFS, this client has removed support for it.

data/lib/mogilefs/httpfile.rb

@@ -1,157 +0,0 @@
-# -*- encoding: binary -*-
-require 'stringio'
-require 'uri'
-require 'mogilefs/backend'
-require 'mogilefs/util'
-
-##
-# HTTPFile wraps up the new file operations for storing files onto an HTTP
-# storage node.
-#
-# You really don't want to create an HTTPFile by hand.  Instead you want to
-# create a new file using MogileFS::MogileFS.new_file.
-#
-#--
-# TODO dup'd content in MogileFS::NFSFile
-
-class MogileFS::HTTPFile < StringIO
-  include MogileFS::Util
-
-  class EmptyResponseError < MogileFS::Error; end
-  class BadResponseError < MogileFS::Error; end
-  class UnparseableResponseError < MogileFS::Error; end
-  class NoStorageNodesError < MogileFS::Error
-    def message; 'Unable to open socket to storage node'; end
-  end
-
-  ##
-  # The URI this file will be stored to.
-
-  attr_reader :uri
-
-  ##
-  # The key for this file.  This key won't represent a real file until you've
-  # called #close.
-
-  attr_reader :key
-
-  ##
-  # The class of this file.
-
-  attr_reader :class
-
-  ##
-  # The big_io name in case we have file > 256M
-
-  attr_accessor :big_io
-
-  attr_accessor :streaming_io
-
-  ##
-  # Works like File.open.  Use MogileFS::MogileFS#new_file instead of this
-  # method.
-
-  def self.open(*args)
-    fp = new(*args)
-    fp.set_encoding(Encoding::BINARY) if fp.respond_to?(:set_encoding)
-
-    return fp unless block_given?
-
-    begin
-      yield fp
-    ensure
-      fp.close
-    end
-  end
-
-  ##
-  # Creates a new HTTPFile with MogileFS-specific data.  Use
-  # MogileFS::MogileFS#new_file instead of this method.
-
-  def initialize(mg, fid, klass, key, dests, content_length)
-    super ''
-    @mg = mg
-    @fid = fid
-    @uri = @devid = nil
-    @klass = klass
-    @key = key
-    @big_io = nil
-    @streaming_io = nil
-
-    @dests = dests
-    @tried = {}
-
-    @socket = nil
-  end
-
-  ##
-  # Writes an HTTP PUT request to +sock+ to upload the file and
-  # returns file size if the socket finished writing
-  def upload(devid, uri)
-    file_size = length
-    sock = Socket.mogilefs_new(uri.host, uri.port)
-
-    if @streaming_io
-      file_size = @streaming_io.length
-      syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
-                          "Content-Length: #{file_size}\r\n\r\n")
-      @streaming_io.call(Proc.new do |data_to_write|
-        syswrite_full(sock, data_to_write)
-      end)
-    elsif @big_io
-      # Don't try to run out of memory
-      File.open(@big_io, "rb") do |fp|
-        file_size = fp.stat.size
-        syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
-                            "Content-Length: #{file_size}\r\n\r\n")
-        sysrwloop(fp, sock)
-      end
-    else
-      syswrite_full(sock, "PUT #{uri.request_uri} HTTP/1.0\r\n" \
-                          "Content-Length: #{length}\r\n\r\n#{string}")
-    end
-
-    line = sock.gets or
-      raise EmptyResponseError, 'Unable to read response line from server'
-
-    if line =~ %r%^HTTP/\d+\.\d+\s+(\d+)% then
-      case $1.to_i
-      when 200..299 then # success!
-      else
-        raise BadResponseError, "HTTP response status from upload: #{$1}"
-      end
-    else
-      raise UnparseableResponseError, "Response line not understood: #{line}"
-    end
-
-    @mg.backend.create_close(:fid => @fid, :devid => devid,
-                             :domain => @mg.domain, :key => @key,
-                             :path => uri.to_s, :size => file_size)
-    file_size
-  end # def upload
-
-  def close
-    try_dests = @dests.dup
-    last_err = nil
-
-    loop do
-      devid, url = try_dests.shift
-      devid && url or break
-
-      uri = URI.parse(url)
-      begin
-        bytes = upload(devid, uri)
-        @devid, @uri = devid, uri
-        return bytes
-      rescue SystemCallError, Errno::ECONNREFUSED, MogileFS::Timeout,
-             EmptyResponseError, BadResponseError,
-             UnparseableResponseError => err
-        last_err = @tried[url] = err
-      end
-    end
-
-    raise last_err ? last_err : NoStorageNodesError
-  end
-
-end
-