mogilefs-client 1.3.1 → 2.0.0

data/.gitignore

@@ -0,0 +1,2 @@
+/pkg
+/doc

data/GNUmakefile

@@ -0,0 +1,32 @@
+# use GNU Make to run tests in parallel, and without depending on Rubygems
+all:: test
+
+T := $(wildcard test/test*.rb)
+TO := $(subst .rb,.log,$(T))
+
+test: $(T)
+	@cat $(TO) | ruby test/aggregate.rb
+	@$(RM) $(TO)
+clean:
+	$(RM) $(TO) $(addsuffix +,$(TO))
+
+t = $(basename $(notdir $@))
+t_log = $(subst .rb,.log,$@)
+
+$(T):
+	@echo $(t); ruby -I lib $@ $(TEST_OPTS) > $(t_log)+ 2>&1
+	@mv $(t_log)+ $(t_log)
+
+# using make instead of rake since Rakefile takes too long to load
+Manifest.txt:
+	git ls-files > $@+
+	mv $@+ $@
+
+libs := $(wildcard lib/*.rb lib/*/*.rb)
+flay_flags =
+flog_flags =
+flay: $(libs)
+	flay $(flay_flags) $^
+flog: $(libs)
+	flog $(flog_flags) $^
+.PHONY: $(T) Manifest.txt

data/History.txt

@@ -1,3 +1,26 @@
+= 2.0.0
+
+* use a set of standard exceptions based on MogileFS::Error,
+  this is an incompatible API change (hence the 2.0.0 version number).
+* remove NFS support since it's gone in MogileFS 2.x and NFS is horrible
+* mog timeouts and retries increased
+* more consistent handling of bad sockets, all sockets used internally
+  are now explicitly non-blocking and IO.select is used for timeouts
+  instead of using threads behind our backs
+* remove open-uri and net/http dependencies, they were bad with large files
+* add paths_size() method, allowing single mogilefsd call to get size and paths
+* add read-only MogileFS::Mysql driver, allowing mogilefsd to be bypassed
+* use TCP_NODELAY when available on sockets that require low latency
+* use TCP_CORK on bulk transfers to improve bandwidth usage
+* better HTTP error handling
+* verify_uris method in new MogileFS::Network module which allows
+  async verification of several URIs at once on the client side.
+* handle multiple device failover correctly on HTTP uploads
+* initial big_file read support (should be mogtool(1)-compatible)
+* unit tests can be run in parallel using GNU Make, 3x faster on a Core2 Duo
+* unit tests modified to use real sockets for easier verification of
+  timeout and error condition handling.
+
 = 1.3.1
 
 * Fix missing MogileFS::Util include for sysrwloop in MogileFS::MogileFS

data/LICENSE.txt

@@ -1,4 +1,5 @@
 Copyright 2005 Eric Hodel, The Robot Co-op.  All rights reserved.
+Copyright 2008 Eric Wong.  All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions

data/Manifest.txt

@@ -1,3 +1,5 @@
+.gitignore
+GNUmakefile
 History.txt
 LICENSE.txt
 Manifest.txt
@@ -7,15 +9,24 @@ bin/mog
 lib/mogilefs.rb
 lib/mogilefs/admin.rb
 lib/mogilefs/backend.rb
+lib/mogilefs/bigfile.rb
 lib/mogilefs/client.rb
 lib/mogilefs/httpfile.rb
 lib/mogilefs/mogilefs.rb
-lib/mogilefs/nfsfile.rb
+lib/mogilefs/mysql.rb
+lib/mogilefs/network.rb
 lib/mogilefs/pool.rb
 lib/mogilefs/util.rb
+test/.gitignore
+test/aggregate.rb
 test/setup.rb
 test/test_admin.rb
 test/test_backend.rb
+test/test_bigfile.rb
 test/test_client.rb
+test/test_db_backend.rb
 test/test_mogilefs.rb
+test/test_mysql.rb
+test/test_network.rb
 test/test_pool.rb
+test/test_util.rb

data/README.txt

@@ -14,10 +14,20 @@ File bugs:
 
 http://rubyforge.org/tracker/?func=add&group_id=1513&atid=5921
 
+Source repository (git):
+
+git://git.bogomips.org/mogilefs-client.git
+
+http://git.bogomips.org/mogilefs-client.git
+
+Repository browser (cgit):
+
+http://git.bogomips.org/cgit/mogilefs-client.git
+
 == About
 
 A Ruby MogileFS client.  MogileFS is a distributed filesystem written
-by Danga Interactive.  This client supports NFS and HTTP modes.
+by Danga Interactive.  This client only supports HTTP.
 
 For information on MogileFS see:
 
@@ -25,7 +35,8 @@ http://danga.com/mogilefs/
 
 == Installing mogilefs-client
 
-First you need a MogileFS setup.  You can find information on how to do that at the above URL.
+First you need a MogileFS setup.  You can find information on how to do
+that at the above URL.
 
 Then install the gem:
 
@@ -35,32 +46,30 @@ Then install the gem:
 
   # Create a new instance that will communicate with these trackers:
   hosts = %w[192.168.1.69:6001 192.168.1.70:6001]
-  mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts
-                              :root => '/mnt/mogilefs')
-  
+  mg = MogileFS::MogileFS.new(:domain => 'test', :hosts => hosts)
+
   # Stores "A bunch of text to store" into 'some_key' with a class of 'text'.
   mg.store_content 'some_key', 'text', "A bunch of text to store"
-  
+
   # Retrieve data from 'some_key'
   data = mg.get_file_data 'some_key'
-  
+
   # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
   # 'image'.
   mg.store_file 'my_image', 'image', 'image.jpeg'
-  
+
   # Store the contents of 'image.jpeg' into the key 'my_image' with a class of
   # 'image' using an open IO.
   File.open 'image.jpeg' do |fp|
     mg.store_file 'my_image', 'image', fp
   end
-  
+
   # Remove the key 'my_image' and 'some_key'.
   mg.delete 'my_image'
   mg.delete 'some_key'
 
 == WARNING!
 
-This client is only known to work in NFS mode.  HTTP mode is implemented but
-only lightly tested in production environments.  If you find a bug,
-please report it on the Rubyforge project.
-
+This client is only supported in HTTP mode.  NFS mode was previously
+supported in 1.3.x, but since MogileFS 2.x has dropped support for
+NFS, this client has removed support for it.

data/bin/mog

@@ -14,6 +14,8 @@ def parse_config_file!(path, overwrite = false)
       dest[$1.to_sym] = $2
     elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
       dest[:hosts] = $1.split(/\s*,\s*/)
+    elsif m = /^timeout\s*=\s*(.*)/.match(line)
+      dest[:timeout] = m[1].to_f
     else
       STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
     end
@@ -31,6 +33,7 @@ config_file = nil
 ls_l = false
 ls_h = false
 test = {}
+cat = { :raw => false }
 
 ARGV.options do |x|
   x.banner = "Usage: #{$0} [options] <command> [<arguments>]"
@@ -45,6 +48,7 @@ ARGV.options do |x|
   end
 
   x.on('-e', 'True if key exists') { test[:e] = true }
+  x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
 
   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
@@ -67,7 +71,7 @@ env_cfg = {}
 if ENV["MOG_TRACKERS"]
   env_cfg[:hosts] = ENV["MOG_TRACKERS"].split(/\s*,\s*/)
 end
-if ENV["MOG_HOSTS"] && env_cfg[:hosts].empty?
+if ENV["MOG_HOSTS"] && (env_cfg[:hosts] || []).empty?
   env_cfg[:hosts] = ENV["MOG_HOSTS"].split(/\s*,\s*/)
 end
 env_cfg[:domain] = ENV["MOG_DOMAIN"] if ENV["MOG_DOMAIN"]
@@ -91,20 +95,42 @@ unless cmd = ARGV.shift
   exit 1
 end
 
+cfg[:timeout] ||= 30 # longer timeout for interactive use
 include MogileFS::Util
 mg = MogileFS::MogileFS.new(cfg)
 
+def store_file_retry(mg, key, storage_class, filepath)
+  tries = 0
+  begin
+    mg.store_file(key, storage_class, filepath)
+  rescue MogileFS::UnreadableSocketError,
+         MogileFS::Backend::NoDevicesError => err
+    if ((tries += 1) < 10)
+      STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
+      retry
+    else
+      STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
+    end
+    exit 1
+  end
+end
+
 begin
   case cmd
   when 'cp'
     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
     key = ARGV.shift or raise ArgumentError, '<filename> <key>'
     ARGV.shift and raise ArgumentError, '<filename> <key>'
-    cfg[:class] or raise ArgumentError, 'E: --class must be specified'
-    mg.store_file(key, cfg[:class], filename)
+    store_file_retry(mg, key, cfg[:class], filename)
   when 'cat'
     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
-    ARGV.each { |key| mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) } }
+    ARGV.each do |key|
+      if (!cat[:raw] && key =~ /^_big_info:/)
+        mg.bigfile_write(key, STDOUT, {:verify => true})
+      else
+        mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) }
+      end
+    end
   when 'ls'
     prefixes = ARGV.empty? ? [ nil ] : ARGV
     prefixes.each do |prefix|
@@ -160,9 +186,14 @@ begin
     buf = ''
     tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
     at_exit { tmp.unlink }
+
+    # if stdout is pointing to /dev/null, don't bother installing the filter.
+    STDOUT.sync = true
+    tee_filter = File.stat('/dev/null') == STDOUT.stat ?
+                 nil : Proc.new { |buf| STDOUT.write(buf); buf }
     begin
-      sysrwloop(STDIN, tmp)
-      mg.store_file(key, cfg[:class], tmp.path)
+      sysrwloop(STDIN, tmp, tee_filter)
+      store_file_retry(mg, key, cfg[:class], tmp.path)
     ensure
       tmp.close
     end

data/lib/mogilefs.rb

@@ -6,21 +6,35 @@
 
 module MogileFS
 
-  VERSION = '1.3.1'
+  VERSION = '2.0.0'.freeze
 
   ##
   # Raised when a socket remains unreadable for too long.
 
-  class UnreadableSocketError < RuntimeError; end
+  class Error < StandardError; end
+  class UnreadableSocketError < Error; end
+  class SizeMismatchError < Error; end
+  class ChecksumMismatchError < RuntimeError; end
+  class ReadOnlyError < Error
+    def message; 'readonly mogilefs'; end
+  end
+  class EmptyPathError < Error
+    def message; 'Empty path for mogile upload'; end
+  end
+
+  class UnsupportedPathError < Error; end
+  class RequestTruncatedError < Error; end
+  class InvalidResponseError < Error; end
+  class UnreachableBackendError < Error
+    def message; "couldn't connect to mogilefsd backend"; end
+  end
 
 end
 
-require 'socket'
-
 require 'mogilefs/backend'
-require 'mogilefs/nfsfile'
 require 'mogilefs/httpfile'
 require 'mogilefs/client'
+require 'mogilefs/bigfile'
 require 'mogilefs/mogilefs'
 require 'mogilefs/admin'
 

data/lib/mogilefs/admin.rb

@@ -40,9 +40,8 @@ class MogileFS::Admin < MogileFS::Client
   #     "altmask"=>""}]
 
   def get_hosts(hostid = nil)
-    args = hostid ? { :hostid => hostid } : {}
-    res = @backend.get_hosts args
-    return clean('hosts', 'host', res)
+    clean('hosts', 'host',
+          @backend.get_hosts(hostid ? { :hostid => hostid } : {}))
   end
 
   ##
@@ -62,9 +61,8 @@ class MogileFS::Admin < MogileFS::Client
   #     "mb_total"=>""}]
 
   def get_devices(devid = nil)
-    args = devid ? { :devid => devid } : {}
-    res = @backend.get_devices args
-    return clean('devices', 'dev', res)
+    clean('devices', 'dev',
+          @backend.get_devices(devid ? { :devid => devid } : {}))
   end
 
   ##
@@ -88,8 +86,8 @@ class MogileFS::Admin < MogileFS::Client
   #     "key"=>"new_new_key"}]
 
   def list_fids(from_fid, to_fid)
-    res = @backend.list_fids :from => from_fid, :to => to_fid
-    return clean('fid_count', 'fid_', res)
+    clean('fid_count', 'fid_',
+          @backend.list_fids(:from => from_fid, :to => to_fid))
   end
 
   ##
@@ -126,7 +124,7 @@ class MogileFS::Admin < MogileFS::Client
     stats.delete 'file' if stats['file'].empty?
     stats.delete 'replication' if stats['replication'].empty?
 
-    return stats
+    stats
   end
 
   ##
@@ -148,25 +146,24 @@ class MogileFS::Admin < MogileFS::Client
       domains[res["domain#{i}"]] = Hash[*domain.flatten]
     end
 
-    return domains
+    domains
   end
 
   ##
   # Creates a new domain named +domain+.  Returns nil if creation failed.
 
   def create_domain(domain)
-    raise 'readonly mogilefs' if readonly?
+    raise MogileFS::ReadOnlyError if readonly?
     res = @backend.create_domain :domain => domain
-    return res['domain'] unless res.nil?
+    res ? res['domain'] : nil
   end
 
   ##
   # Deletes +domain+.  Returns true if successful, false if not.
 
   def delete_domain(domain)
-    raise 'readonly mogilefs' if readonly?
-    res = @backend.delete_domain :domain => domain
-    return !res.nil?
+    raise MogileFS::ReadOnlyError if readonly?
+    ! @backend.delete_domain(:domain => domain).nil?
   end
 
   ##
@@ -174,7 +171,7 @@ class MogileFS::Admin < MogileFS::Client
   # +mindevcount+ devices.  Returns nil on failure.
 
   def create_class(domain, klass, mindevcount)
-    return modify_class(domain, klass, mindevcount, :create)
+    modify_class(domain, klass, mindevcount, :create)
   end
 
   ##
@@ -182,7 +179,7 @@ class MogileFS::Admin < MogileFS::Client
   # devices.  Returns nil on failure.
 
   def update_class(domain, klass, mindevcount)
-    return modify_class(domain, klass, mindevcount, :update)
+    modify_class(domain, klass, mindevcount, :update)
   end
 
   ##
@@ -190,8 +187,7 @@ class MogileFS::Admin < MogileFS::Client
   # not.
 
   def delete_class(domain, klass)
-    res = @backend.delete_class :domain => domain, :class => klass
-    return !res.nil?
+    ! @backend.delete_class(:domain => domain, :class => klass).nil?
   end
 
   ##
@@ -202,23 +198,22 @@ class MogileFS::Admin < MogileFS::Client
     raise ArgumentError, "Must specify ip and port" unless \
       args.include? :ip and args.include? :port
 
-    return modify_host(host, args, 'create')
+    modify_host(host, args, 'create')
   end
 
   ##
   # Updates +host+ with +args+.  Returns true if successful, false if not.
 
   def update_host(host, args = {})
-    return modify_host(host, args, 'update')
+    modify_host(host, args, 'update')
   end
 
   ##
   # Deletes host +host+.  Returns nil on failure.
 
   def delete_host(host)
-    raise 'readonly mogilefs' if readonly?
-    res = @backend.delete_host :host => host
-    return !res.nil?
+    raise MogileFS::ReadOnlyError if readonly?
+    ! @backend.delete_host(:host => host).nil?
   end
 
   ##
@@ -226,9 +221,8 @@ class MogileFS::Admin < MogileFS::Client
   # 'alive', 'down', or 'dead'.
 
   def change_device_state(host, device, state)
-    raise 'readonly mogilefs' if readonly?
-    res = @backend.set_state :host => host, :device => device, :state => state
-    return !res.nil?
+    raise MogileFS::ReadOnlyError if readonly?
+    ! @backend.set_state(:host => host, :device => device, :state => state).nil?
   end
 
   protected unless defined? $TESTING
@@ -238,11 +232,11 @@ class MogileFS::Admin < MogileFS::Client
   # +action+.  Returns the class name if successful, nil if not.
 
   def modify_class(domain, klass, mindevcount, action)
-    raise 'readonly mogilefs' if readonly?
+    raise MogileFS::ReadOnlyError if readonly?
     res = @backend.send("#{action}_class", :domain => domain, :class => klass,
                                           :mindevcount => mindevcount)
 
-    return res['class'] unless res.nil?
+    res ? res['class'] : nil
   end
 
   ##
@@ -251,8 +245,7 @@ class MogileFS::Admin < MogileFS::Client
 
   def modify_host(host, args = {}, action = 'create')
     args[:host] = host
-    res = @backend.send "#{action}_host", args
-    return !res.nil?
+    ! @backend.send("#{action}_host", args).nil?
   end
 
   ##
@@ -271,9 +264,9 @@ class MogileFS::Admin < MogileFS::Client
   #          "host1_status"=>"alive",
   #          "host1_altmask"=>""}
   #   admin.clean 'hosts', 'host', res
-  # 
+  #
   # Returns:
-  # 
+  #
   #   [{"status"=>"alive",
   #     "http_get_port"=>"",
   #     "http_port"=>"",
@@ -286,7 +279,7 @@ class MogileFS::Admin < MogileFS::Client
 
   def clean(count, prefix, res, underscore = true)
     underscore = underscore ? '_' : ''
-    return (1..res[count].to_i).map do |i|
+    (1..res[count].to_i).map do |i|
       dev = res.select { |k,_| k =~ /^#{prefix}#{i}#{underscore}/ }.map do |k,v|
         [k.sub(/^#{prefix}#{i}#{underscore}/, ''), v]
       end