mogilefs-client 3.1.1 → 3.2.0.rc1

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.1.1"
+DEF_VER = "v3.2.0-rc1"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/Manifest.txt

@@ -0,0 +1,67 @@
+.document
+.gitignore
+.wrongdoc.yml
+GIT-VERSION-GEN
+GNUmakefile
+HACKING
+History
+LICENSE
+README
+Rakefile
+TODO
+bin/mog
+examples/stale_fid_checker.rb
+lib/mogilefs.rb
+lib/mogilefs/admin.rb
+lib/mogilefs/backend.rb
+lib/mogilefs/bigfile.rb
+lib/mogilefs/bigfile/filter.rb
+lib/mogilefs/chunker.rb
+lib/mogilefs/client.rb
+lib/mogilefs/copy_stream.rb
+lib/mogilefs/http_file.rb
+lib/mogilefs/http_reader.rb
+lib/mogilefs/mogilefs.rb
+lib/mogilefs/mysql.rb
+lib/mogilefs/new_file.rb
+lib/mogilefs/new_file/common.rb
+lib/mogilefs/new_file/content_range.rb
+lib/mogilefs/new_file/stream.rb
+lib/mogilefs/new_file/tempfile.rb
+lib/mogilefs/new_file/writer.rb
+lib/mogilefs/paths_size.rb
+lib/mogilefs/pool.rb
+lib/mogilefs/socket.rb
+lib/mogilefs/socket/kgio.rb
+lib/mogilefs/socket/pure_ruby.rb
+lib/mogilefs/socket_common.rb
+lib/mogilefs/util.rb
+setup.rb
+test/.gitignore
+test/aggregate.rb
+test/exec.rb
+test/fresh.rb
+test/integration.rb
+test/setup.rb
+test/socket_test.rb
+test/test_admin.rb
+test/test_backend.rb
+test/test_bigfile.rb
+test/test_client.rb
+test/test_db_backend.rb
+test/test_fresh.rb
+test/test_http_reader.rb
+test/test_mogilefs.rb
+test/test_mogilefs_integration.rb
+test/test_mogilefs_integration_large_pipe.rb
+test/test_mogilefs_integration_list_keys.rb
+test/test_mogilefs_socket_kgio.rb
+test/test_mogilefs_socket_pure.rb
+test/test_mogstored_rack.rb
+test/test_mogtool_bigfile.rb
+test/test_mysql.rb
+test/test_pool.rb
+Manifest.txt
+ChangeLog
+NEWS
+lib/mogilefs/version.rb

data/NEWS

@@ -0,0 +1,308 @@
+=== Ruby mogilefs-client 3.2.0-rc1 / 2012-06-15 01:05 UTC
+
+  * "list_keys" and "exist?" client commands raise errors properly
+    on failure.
+
+  * backend connections no longer terminate on ERR responses, only
+    on socket/connection errors.
+
+  * support the "updateclass" client command.  This is for updating
+    the class of a given key and not to be confused with the
+    "update_class" admin command.
+
+  * "new_file" checksum usage is now documented since MogileFS 2.60
+    includes official support for checksums
+
+=== Ruby mogilefs-client 3.1.1 / 2012-02-28 22:12 UTC
+
+  This releases fixes problems short reads when slurping files
+  into memory.  Thanks to Matthew Draper for this fix.
+  There are also minor documentation updates.
+
+=== Ruby mogilefs-client 3.1.0 / 2011-12-15 04:10 UTC
+
+  * improved API support for uploading large files
+    While we've always supported uploading large files, the
+    (still-supported) existing APIs were somewhat awkward or
+    required the file to exist on the file system.  See
+    MogileFS::NewFile for details and examples.
+
+  * more informative exception messages for timed-out requests
+
+  * :fail_timeout parameter, the timeout for retrying a failed
+    tracker connection.  This defaults to 5 seconds (same as
+    previous versions where this was hard-coded.
+
+  * :new_file_max_time parameter
+    Controls the maximum of time spent creating and uploading
+    a new file in MogileFS.   This defaults to 1 hour (which
+    matching the expiry time of a row in the MogileFS internal
+    tempfile table).
+
+  * store_file works on unlinked File/Tempfile objects
+
+  * each_fid method in MogileFS::Admin fixed
+
+  * stale_fid_checker example script added
+
+  * mogstored_rack example split into a standalone RubyGem:
+    http://bogomips.org/mogstored_rack/
+
+  * backend error constants are generated on const_missing,
+    instead of when raised, making it easier to rescue exceptions
+    we didn't explicitly enable
+
+  * some internal cleanups and documentation improvements
+
+=== Ruby mogilefs-client 3.0.0 / 2011-11-28 20:53 UTC
+
+  Changes since 3.0.0-rc1:
+
+  * 1.8 copy_stream emulator respects creation umask
+    This matches the latest IO.copy_stream behavior.
+    [ruby-core:41308], r33851 in ruby/trunk
+
+  * higher timeouts for uploads to compensate for slow
+    (but not dead) servers.
+
+  Changes since v2.2.0:
+
+  === client changes
+
+  * "store_file" now accepts any IO object capable of streaming
+    data (e.g. pipes and sockets).   This uses chunked
+    Transfer-Encoding for PUTs, so backend storage nodes
+    will need to support this (latest mogstored does).
+
+  * "store_file" no longer uses an infinite timeout when
+    awaiting a response after a PUT, the new timeout for the
+    response is now calculated based on the time and size of
+    the PUT request.
+
+  * new commands: "file_debug" and "file_info" (new commands
+    in mogilefsd, be sure you have the latest version)
+
+  * "get_paths" takes optional ":pathcount" parameter
+    to control the number of returned paths.
+
+  * "get_file_data" supports offset and count for partial
+    transfer (requires support from storage node for
+    Range: requests, most HTTP servers are capable of this)
+
+  * IO.copy_stream is enabled by default under Ruby 1.9.3.
+    Expect performance improvements.
+
+  * "list_keys" with a passed block (for additional info) is
+    faster due to internal pipelining implementation and the
+    addition of "file_info" support.
+
+  * fixed handling of "+" in key/domain names (old bug)
+
+  * rare, truncated partial responses due to network/server
+    failure now raise MogileFS::InvalidResponseError
+
+  === admin changes
+
+  Admin support is still a work-in-progress,
+  I usually just find myself using "mogadm" anyways.
+
+  * new admin commands: "replicate_now"
+
+  * "get_stats" no longer works on new mogilefsd versions
+
+  * get_domains handles "repl_policy" field correctly for classes
+    in MogileFS 2.x
+
+  * admin commands should convert all numeric fields to either
+    Integer or Float objects and not String representations
+    of numerics.  Affected methods include:
+     get_hosts, get_devices, list_fids, each_fids, get_domains
+
+  === miscellany
+
+  * we no longer add methods to standard Ruby classes
+    (at least we never /changed/ existing methods :P)
+
+  * {kgio}[http://bogomips.org/kgio] automatically used
+    if available, but not required to avoid exceptions
+    with non-blocking I/O
+
+  * dropped Ruby 1.8.6 support, 1.8.7 or later is required.
+
+  * Tested with MRI 1.8.7, 1.9.3 and Rubinius 1.2.4
+
+  * MogileFS::Mysql - deprecated, to be removed in 2012
+
+  * improved test suite
+
+  * some optional experimental features/changes,
+    see "git log" for details
+
+  * Added internal pipelining implementation, this is
+    not easy-to-use since the server can respond-out-of-order,
+    but still useful for things like a list_keys+file_info
+    loop.
+
+=== mogilefs-client 3.0.0-rc1 / 2011-11-21 02:48 UTC
+
+  === client changes
+
+  * "store_file" now accepts any IO object capable of streaming
+    data (e.g. pipes and sockets).   This uses chunked
+    Transfer-Encoding for PUTs, so backend storage nodes
+    will need to support this (latest mogstored does).
+
+  * "store_file" no longer uses an infinite timeout when
+    awaiting a response after a PUT, the new timeout for the
+    response is now calculated based on the time and size of
+    the PUT request.
+
+  * new commands: "file_debug" and "file_info" (new commands
+    in mogilefsd, be sure you have the latest version)
+
+  * "get_paths" takes optional ":pathcount" parameter
+    to control the number of returned paths.
+
+  * "get_file_data" supports offset and count for partial
+    transfer (requires support from storage node for
+    Range: requests, most HTTP servers are capable of this)
+
+  * IO.copy_stream is enabled by default under Ruby 1.9.3.
+    Expect performance improvements.
+
+  * "list_keys" with a passed block (for additional info) is
+    faster due to internal pipelining implementation and the
+    addition of "file_info" support.
+
+  * fixed handling of "+" in key/domain names (old bug)
+
+  * rare, truncated partial responses due to network/server
+    failure now raise MogileFS::InvalidResponseError
+
+  === admin changes
+
+  Admin support is still a work-in-progress,
+  I usually just find myself using "mogadm" anyways.
+
+  * new admin commands: "replicate_now"
+
+  * "get_stats" no longer works on new mogilefsd versions
+
+  * get_domains handles "repl_policy" field correctly for classes
+    in MogileFS 2.x
+
+  * admin commands should convert all numeric fields to either
+    Integer or Float objects and not String representations
+    of numerics.  Affected methods include:
+     get_hosts, get_devices, list_fids, each_fids, get_domains
+
+  === miscellany
+
+  * we no longer add methods to standard Ruby classes
+    (at least we never /changed/ existing methods :P)
+
+  * {kgio}[http://bogomips.org/kgio] automatically used
+    if available, but not required to avoid exceptions
+    with non-blocking I/O
+
+  * dropped Ruby 1.8.6 support, 1.8.7 or later is required.
+
+  * Tested with MRI 1.8.7, 1.9.3 and Rubinius 1.2.4
+
+  * MogileFS::Mysql - deprecated, to be removed in 2012
+
+  * improved test suite
+
+  * some optional experimental features/changes,
+    see "git log" for details
+
+  * Added internal pipelining implementation, this is
+    not easy-to-use since the server can respond-out-of-order,
+    but still useful for things like a list_keys+file_info
+    loop.
+
+=== mogilefs-client 2.2.0 / 2011-01-13 02:10 UTC
+
+  = 2.2.0
+  * internal cleanups (no public API breakage)
+  * refactor backend socket/connection handling for reliability
+
+  There'll probably be more aggressive cleanups/refactoring in
+  future releases if I have time.
+
+=== mogilefs-client 2.1.0 / 2009-04-12 03:04 UTC
+
+  = 2.1.0
+  * MySQL interface returns integer length and devcount (API change)
+  * Ensure store_{content,file} always returns size (API fix)
+  * Add get_uris API method
+  * Respect timeout when doing get_file_data
+  * MySQL interface filters out URLs for down/dead hosts/devices
+  * Really remove all NFS support
+  * get_file in slurp mode slurps all output correctly
+
+  Eric Wong (18):
+        tests: retry random ports correctly
+        test_mogilefs: fix race conditions
+        MySQL interface returns integer length and devcount
+        Respect timeout when doing get_file_data
+        Unify internal HTTP GET/HEAD methods
+        Close socket we create before raising exceptions
+        Compact get_paths output so we don't have nils
+        Really remove all NFS support
+        README: add a note about emailing me
+        Extra checking for full_timeout in sysread_full
+        Add get_uris API method
+        verify_uris: use write_nonblock instead of syswrite
+        README: add links to the repo.or.cz mirror
+        mysql: filter out URLs for down/dead hosts/devices
+        mog: small cleanup
+        Ensure store_{content,file} always returns size
+        GNUmakefile: better logging/output control
+        mogilefs-client 2.1.0
+
+  godfat (1):
+        call IO.select([sock]) if sock is not ready.
+
+=== mogilefs-client 1.2.1 / 2009-03-25 07:29 UTC
+
+  Retroactively tagging this old release since
+  it was in SVN or Perforce...
+
+=== mogilefs-client 2.0.2 / 2009-03-02 21:56 UTC
+
+  initial
+
+=== mogilefs-client v2.0.1 / 2009-02-25 08:13 UTC
+
+  * verify_uris method made more robust
+  * preliminary Ruby 1.9 compatibility, tests still need some work
+  * allow store_content to be used with a streamable object of known length
+  * add setup.rb for non-RubyGems users
+
+=== mogilefs-client 2.0.0 / 2009-02-03 04:27 UTC
+
+  initial
+
+=== v1.3.1 / 2008-10-03 03:01 UTC
+
+  * Fix missing MogileFS::Util include for sysrwloop in MogileFS::MogileFS
+
+=== v1.3.0 / 2008-09-18 23:46 UTC
+
+  * Fixed MogileFS#rename.  Bug #14465 submitted by Justin Dossey.
+  * Removed infinite loop in MogileFS::HTTPFile#store_file.  Patch #13789
+    submitted by  Andy Lo-A-Foe.
+  * Made MogileFS#get_file_data timeout configurable.  Bug #13490 submitted by
+    Andy Lo-A-Foe.
+  * Add MogileFS#size.  Feature Request #14484 submitted by Justin Dossey.
+  * Fix MogileFS#get_file_data to return the data for HTTP mode.  Bug #7133
+    submitted by John Wanko.
+  * New maintainer: Eric Wong
+  * Add `mog' command-line tool as a demo/example
+  * Lower memory consumption with large files
+  * Allow get_file_data to accept a block for large files
+  * Fix each_keys loop termination condition
+  * Apply error handling patch from Matthew Willson.  Bug #15987
+  * Merge large file patch from Andy Lo-A-Foe.  Bug #13764
+

data/README

@@ -14,6 +14,7 @@ email :: mailto:normalperson@yhbt.net
 repo :: git://bogomips.org/mogilefs-client.git
 cgit :: http://bogomips.org/mogilefs-client.git
 gitweb :: http://repo.or.cz/w/ruby-mogilefs-client.git
+download :: http://bogomips.org/mogilefs-client/files/
 
 == Install
 

data/Rakefile

@@ -7,11 +7,21 @@ $:.unshift 'lib'
 require 'mogilefs'
 Hoe.plugin :seattlerb
 
+if ! File.exist?("ChangeLog") || ! File.exist?("NEWS")
+  system("wrongdoc all")
+end
+
 manifest = "Manifest.txt"
 if ! File.exist?(manifest) ||
-   File.stat(manifest).mtime < File.stat(RVF).mtime
+   File.stat(manifest).mtime < File.stat(RVF).mtime ||
+   File.stat(manifest).mtime < File.stat(__FILE__).mtime
   system("git ls-files > #{manifest}")
-  File.open(manifest, "a") { |fp| fp.puts("lib/mogilefs/version.rb") }
+  File.open(manifest, "a") do |fp|
+    fp.puts "Manifest.txt"
+    fp.puts "ChangeLog"
+    fp.puts "NEWS"
+    fp.puts "lib/mogilefs/version.rb"
+  end
 end
 
 Hoe.spec 'mogilefs-client' do
@@ -19,7 +29,7 @@ Hoe.spec 'mogilefs-client' do
   developer 'Eric Wong', 'normalperson@yhbt.net'
   # developer 'drbrain@segment7.net', 'Eric Hodel'
   self.readme_file = "README"
-  self.history_file = "History"
+  self.history_file = "NEWS"
   self.url = "http://bogomips.org/mogilefs-client"
   self.description = self.paragraphs_of("README", 1)
   self.summary = "MogileFS client library for Ruby"
@@ -36,6 +46,3 @@ task :fix_perms do
     end
   end
 end
-
-# vim: syntax=Ruby
-

data/lib/mogilefs/backend.rb

@@ -118,6 +118,7 @@ class MogileFS::Backend
   add_command :delete_domain
   add_command :create_class
   add_command :update_class
+  add_command :updateclass
   add_command :delete_class
   add_command :create_host
   add_command :update_host
@@ -228,11 +229,15 @@ class MogileFS::Backend
   def do_request(cmd, args, idempotent = false)
     no_raise = args.delete(:ruby_no_raise)
     request = make_request(cmd, args)
+    line = nil
     @mutex.synchronize do
       begin
         io = dispatch_unlocked(request)
-        line = io.timed_gets(@timeout) and
-          return parse_response(line, no_raise ? request : nil)
+        line = io.timed_gets(@timeout)
+        break if /\r?\n\z/ =~ line
+
+        line and raise MogileFS::InvalidResponseError,
+                       "Invalid response from server: #{line.inspect}"
 
         idempotent or
           raise EOFError, "end of file reached after: #{request.inspect}"
@@ -251,6 +256,7 @@ class MogileFS::Backend
         shutdown_unlocked(true)
       end while idempotent
     end # @mutex.synchronize
+    parse_response(line, no_raise ? request : nil)
   end
 
   # Makes a new request string for +cmd+ and +args+.

data/lib/mogilefs/mogilefs.rb

@@ -140,7 +140,14 @@ class MogileFS::MogileFS < MogileFS::Client
   # Returns +true+ if +key+ exists, +false+ if not
   def exist?(key)
     args = { :key => key, :domain => @domain , :ruby_no_raise => true}
-    Hash === @backend.get_paths(args)
+    case rv = @backend.get_paths(args)
+    when Hash
+      true
+    when MogileFS::Backend::UnknownKeyError
+      false
+    else
+      raise rv
+    end
   end
 
   # Get the URIs for +key+ (paths) as URI::HTTP objects
@@ -172,6 +179,20 @@ class MogileFS::MogileFS < MogileFS::Client
   # [ :class => String]
   #
   #   The MogileFS storage class of the object.
+  #
+  # [:content_md5 => String, Proc, or :trailer]
+  #
+  #   This can either be a Base64-encoded String, a Proc object, or
+  #   the :trailer symbol.  If given a String, it will be used as the
+  #   Content-MD5 HTTP header.  If given the :trailer symbol, this library
+  #   will automatically generate an Content-MD5 HTTP trailer.  If given
+  #   a Proc object, this Proc object should give a Base64-encoded string
+  #   which can be used as the Content-MD5 HTTP trailer when called at the
+  #   end of the request.
+  #
+  #   Keep in mind most HTTP servers do not support HTTP trailers, so
+  #   passing a String is usually the safest way to use this.
+  #
   def new_file(key, args = nil, bytes = nil) # :yields: file
     raise MogileFS::ReadOnlyError if readonly?
     opts = { :key => key, :multi_dest => 1 }
@@ -256,6 +277,14 @@ class MogileFS::MogileFS < MogileFS::Client
     true
   end
 
+  # Updates +key+ to +newclass+
+  def updateclass(key, newclass)
+    raise MogileFS::ReadOnlyError if readonly?
+
+    @backend.updateclass(:domain => @domain, :key => key, :class => newclass)
+    true
+  end
+
   # Sleeps +duration+, only used for testing
   def sleep(duration) # :nodoc:
     @backend.sleep :duration => duration
@@ -294,6 +323,7 @@ class MogileFS::MogileFS < MogileFS::Client
                              :after => after, :limit => limit,
                              :ruby_no_raise => true)
     MogileFS::Backend::NoneMatchError === res and return
+    raise res if MogileFS::Error === res
 
     keys = (1..res['key_count'].to_i).map { |i| res["key_#{i}"] }
     if block

data/lib/mogilefs/version.rb

@@ -1 +1 @@
-MogileFS::VERSION = '3.1.1'
+MogileFS::VERSION = '3.2.0.rc1'

data/test/test_fresh.rb

@@ -5,4 +5,25 @@ class TestMogFresh < Test::Unit::TestCase
   include TestFreshSetup
   alias setup setup_mogilefs
   alias teardown teardown_mogilefs
+
+  def test_list_keys_invalid_domain
+    add_host_device_domain
+    domain = @domain + ".non-existent"
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => domain
+    assert_raises(MogileFS::Backend::UnregDomainError) do
+      client.list_keys
+    end
+  end
+
+  def test_invalid_key_exists
+    add_host_device_domain
+    domain = @domain + ".non-existent"
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => domain
+    assert_raises(MogileFS::Backend::UnregDomainError) do
+      client.exist?("FOO")
+    end
+
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    assert_equal false, client.exist?("non-existent")
+  end
 end

data/test/test_mogilefs_integration.rb

@@ -290,4 +290,25 @@ class TestMogileFSIntegration < TestMogIntegration
 
     assert_equal "HIHI", @client.get_file_data("unlinked")
   end
+
+  def test_updateclass
+    admin = MogileFS::Admin.new(:hosts => @trackers)
+    admin.create_class(@domain, "one", 1)
+    admin.create_class(@domain, "two", 2)
+    4.times { admin.clear_cache }
+
+    assert_equal 4, @client.store_content("uc", "default", "DATA")
+    assert_equal true, @client.updateclass("uc", "one")
+    assert_equal true, @client.updateclass("uc", "two")
+    assert_raises(MogileFS::Backend::ClassNotFoundError) do
+      @client.updateclass("uc", "wtf")
+    end
+    assert_raises(MogileFS::Backend::InvalidKeyError) do
+      @client.updateclass("nonexistent", "one")
+    end
+
+    @client.delete "uc"
+    admin.delete_class @domain, "one"
+    admin.delete_class @domain, "two"
+  end
 end