mogilefs-client 2.2.0 → 3.0.0.rc1

data/.document

@@ -0,0 +1,11 @@
+History
+ChangeLog
+LICENSE
+README
+HACKING
+TODO
+lib/mogilefs.rb
+lib/mogilefs/admin.rb
+lib/mogilefs/backend.rb
+lib/mogilefs/client.rb
+lib/mogilefs/mogilefs.rb

data/.gitignore

@@ -2,3 +2,7 @@
 /doc
 /.config
 /InstalledFiles
+/ChangeLog
+/NEWS
+/LATEST
+/lib/mogilefs/version.rb

data/.wrongdoc.yml

@@ -0,0 +1,5 @@
+---
+cgit_url: http://bogomips.org/mogilefs-client.git
+git_url: git://bogomips.org/mogilefs-client.git
+rdoc_url: http://bogomips.org/mogilefs-client
+changelog_start: v1.2.1

data/GIT-VERSION-GEN

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+CONSTANT = "MogileFS::VERSION"
+RVF = "lib/mogilefs/version.rb"
+DEF_VER = "v3.0.0-rc1"
+vn = DEF_VER
+
+# First see if there is a version file (included in release tarballs),
+# then try git-describe, then default.
+if File.exist?(".git")
+  describe = `git describe --abbrev=4 HEAD 2>/dev/null`.strip
+  case describe
+  when /\Av[0-9]*/
+    vn = describe
+    system(*%w(git update-index -q --refresh))
+    unless `git diff-index --name-only HEAD --`.chomp.empty?
+      vn << "-dirty"
+    end
+    vn.tr!('-', '.')
+  end
+end
+
+vn = vn.sub!(/\Av/, "")
+new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
+cur_ruby_version = File.read(RVF) rescue nil
+if new_ruby_version != cur_ruby_version
+  File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
+end
+puts vn if $0 == __FILE__

data/GNUmakefile

@@ -1,5 +1,7 @@
 # use GNU Make to run tests in parallel, and without depending on Rubygems
 all:: test
+RSYNC_DEST := bogomips.org:/srv/bogomips/mogilefs-client
+git_version_gen := $(shell ./GIT-VERSION-GEN)
 
 T := $(wildcard test/test*.rb)
 TO := $(subst .rb,.log,$(T))
@@ -31,10 +33,12 @@ $(T): export RUBYLIB := $(CURDIR)/lib:$(RUBYLIB)
 $(T):
 	$(run_test)
 
+RUBY_VERSION_FILE = lib/mogilefs/version.rb
 # using make instead of rake since Rakefile takes too long to load
 manifest: Manifest.txt
 Manifest.txt:
 	git ls-files > $@+
+	echo $(RUBY_VERSION_FILE) >> $@+
 	cmp $@+ $@ || mv $@+ $@
 	$(RM) -f $@+
 
@@ -52,3 +56,43 @@ flay: $(libs)
 flog: $(libs)
 	flog $(flog_flags) $^
 .PHONY: $(T) Manifest.txt
+
+check-warnings:
+	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
+	  do ruby -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
+RSYNC = rsync
+WRONGDOC = wrongdoc
+
+doc:: .document .wrongdoc.yml $(pkg_extra)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(WRONGDOC) all
+	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
+	cd doc && \
+		ln -s README README.txt && \
+		ln -s README.html README_txt.html && \
+		ln -s LICENSE LICENSE.txt && \
+		ln -s LICENSE.html LICENSE_txt.html && \
+		ln -s History History.txt && \
+		ln -s History.html History_txt.html
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	find doc/images -type f | \
+		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+.PHONY: doc .FORCE-GIT-VERSION-FILE

data/HACKING

@@ -0,0 +1,33 @@
+= Hacking mogilefs-client for Ruby
+
+* The latest code is available via git:
+  - git://bogomips.org/mogilefs-client.git
+  - git://repo.or.cz/ruby-mogilefs-client
+
+* Follow conventions set in existing code, don't add hard runtime
+  dependencies outside of the standard Ruby library.
+
+* Do not hesitate to send plain-text(-only) email to Eric Wong at
+  mailto:normalperson@yhbt.net about anything not covered in
+  the documentation or mailing list archives.  General MogileFS
+  topics can go to the public mailing list at mailto:mogile@googlegroups.com
+  You may still email Wong directly if you do not trust the corporation
+  that hosts the public mailing list.
+
+* Use "git format-patch" and "git send-email" for sending patches.
+
+* Use "git request-pull" as a guideline for formatting pull-requests.
+
+* Test with the latest upstream MogileFS and Ruby versions.
+
+* Integration tests exist for setting up a fresh MogileFS instance
+  with a single host and device using a SQLite backend if you
+  have "mogdbsetup", "mogilefsd", and "mogstored" in your PATH.
+
+* Setting MOG_TEST_TRACKERS= to a comma-delimited list of trackers
+  can enable additional integration tests against a live tracker
+  (with multiple devices/hosts).
+
+* Tests may be run in parallel using GNU make:
+
+  make -j6 test

data/{History.txt → History}

@@ -78,4 +78,3 @@
 = 1.2.0
 
 * Changes lost to time.
-

data/{LICENSE.txt → LICENSE}

@@ -25,4 +25,3 @@ BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-

data/Manifest.txt

@@ -1,33 +1,60 @@
+.document
 .gitignore
+.wrongdoc.yml
+GIT-VERSION-GEN
 GNUmakefile
-History.txt
-LICENSE.txt
+HACKING
+History
+LICENSE
 Manifest.txt
-README.txt
+README
 Rakefile
+TODO
 bin/mog
+examples/mogstored_rack.rb
 lib/mogilefs.rb
 lib/mogilefs/admin.rb
 lib/mogilefs/backend.rb
 lib/mogilefs/bigfile.rb
+lib/mogilefs/bigfile/filter.rb
+lib/mogilefs/chunker.rb
 lib/mogilefs/client.rb
-lib/mogilefs/httpfile.rb
+lib/mogilefs/copy_stream.rb
+lib/mogilefs/http_file.rb
+lib/mogilefs/http_reader.rb
 lib/mogilefs/mogilefs.rb
 lib/mogilefs/mysql.rb
-lib/mogilefs/network.rb
+lib/mogilefs/paths_size.rb
 lib/mogilefs/pool.rb
+lib/mogilefs/socket.rb
+lib/mogilefs/socket/kgio.rb
+lib/mogilefs/socket/pure_ruby.rb
+lib/mogilefs/socket_common.rb
 lib/mogilefs/util.rb
 setup.rb
 test/.gitignore
 test/aggregate.rb
+test/exec.rb
+test/fresh.rb
+test/integration.rb
 test/setup.rb
+test/socket_test.rb
 test/test_admin.rb
 test/test_backend.rb
 test/test_bigfile.rb
 test/test_client.rb
 test/test_db_backend.rb
+test/test_fresh.rb
+test/test_http_reader.rb
 test/test_mogilefs.rb
+test/test_mogilefs_integration.rb
+test/test_mogilefs_integration_large_pipe.rb
+test/test_mogilefs_integration_list_keys.rb
+test/test_mogilefs_socket_kgio.rb
+test/test_mogilefs_socket_pure.rb
+test/test_mogstored_rack.rb
+test/test_mogtool_bigfile.rb
 test/test_mysql.rb
-test/test_network.rb
 test/test_pool.rb
-test/test_util.rb
+test/test_unit_mogstored_rack.rb
+lib/mogilefs/version.rb

data/README

@@ -0,0 +1,51 @@
+= mogilefs-client - MogileFS client library for Ruby
+
+A MogileFS client library for Ruby.  MogileFS is an open source
+distributed filesystem, see: http://mogilefs.org for more details.  This
+library allows any Ruby application to read, write and delete files in a
+MogileFS instance.
+
+== Links
+
+rdoc :: http://bogomips.org/mogilefs-client
+mogilefs :: http://mogilefs.org/
+list :: mailto:mogile@googlegroups.com
+email :: mailto:normalperson@yhbt.net
+repo :: git://bogomips.org/mogilefs-client.git
+cgit :: http://bogomips.org/mogilefs-client.git
+gitweb :: http://repo.or.cz/w/ruby-mogilefs-client.git
+
+== Install
+
+First you need a MogileFS 2.x installation.  You can find information on
+how to do that at http://mogilefs.org
+
+Then install the RubyGem:
+
+  $ gem install mogilefs-client
+
+This library supports Ruby 1.8.7 and later, but Ruby 1.9.3 is
+recommended.  No other libraries are required on the client.
+
+== Usage
+
+See MogileFS::MogileFS
+
+== Contact
+
+Feedback (bug reports, user/development dicussion, patches, pull
+requests) are greatly appreciated and handled via email.  We currently
+piggy-back onto the public MogileFS
+{mailing list}[mailto:mogile@googlegroups.com] for feedback.
+
+If you do not want to deal with the corporate host of the MogileFS
+mailing list, or if you wish to keep your issue secret, feel free to
+email {Eric Wong at}[mailto:normalperson@yhbt.net].
+
+Do not expect Eric Wong to read HTML mail under any circumstances.
+
+== WARNING!
+
+This client is only supported in HTTP mode.  NFS mode was previously
+supported in 1.3.x, but since MogileFS 2.x has dropped support for
+NFS, this client has removed support for it.

data/Rakefile

@@ -1,20 +1,20 @@
 require 'rubygems'
 require 'hoe'
+load "./GIT-VERSION-GEN"
 
 $:.unshift 'lib'
 require 'mogilefs'
+Hoe.plugin :seattlerb
 
-Hoe.new 'mogilefs-client', MogileFS::VERSION do |p|
-  p.rubyforge_name = 'seattlerb'
-  p.author = [ 'Eric Wong', 'Eric Hodel' ]
-  p.email = 'normalperson@yhbt.net' # (Eric Wong)
-  # p.email = 'drbrain@segment7.net' # (Eric Hodel)
-  p.summary = p.paragraphs_of('README.txt', 1).first
-  p.description = p.paragraphs_of('README.txt', 9).first
-  p.url = p.paragraphs_of('README.txt', 5).first
-  p.changes = p.paragraphs_of('History.txt', 0..1).join("\n\n")
-
-  p.extra_dev_deps << ['ZenTest', '>= 3.6.1']
+Hoe.spec 'mogilefs-client' do
+  self.rubyforge_name = 'seattlerb'
+  developer 'Eric Wong', 'normalperson@yhbt.net'
+  # developer 'drbrain@segment7.net', 'Eric Hodel'
+  self.readme_file = "README"
+  self.history_file = "History"
+  self.url = "http://bogomips.org/mogilefs-client"
+  self.description = self.paragraphs_of("README", 1)
+  self.summary = "MogileFS client library for Ruby"
 end
 
 task :fix_perms do

data/TODO

@@ -0,0 +1,10 @@
+Patches and pull-requests (by [public]{mailto:mogile@googlegroups.com} or
+[private email]{mailto:normalperson@yhbt.net}) greatly appreciated!
+
+* MogileFS::Admin needs to be fleshed out
+
+* optional Cool.io support
+
+* optional EventMachine support
+
+* manpage for "mog"

data/bin/mog

@@ -1,24 +1,33 @@
 #!/usr/bin/env ruby
 require 'mogilefs'
 require 'optparse'
-[ STDIN, STDOUT, STDERR].each { |io| io.binmode }
+$stdin.binmode
+$stdout.binmode
+$stderr.sync = $stdout.sync = true
 
 trap('INT') { exit 130 }
 trap('PIPE') { exit 0 }
+if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
+  md5_trailer_nodes.split(/\s*,\s*/).each do |host|
+    MogileFS::HTTPFile::MD5_TRAILER_NODES[host] = true
+  end
+end
 
 # this is to be compatible with config files used by the Perl tools
 def parse_config_file!(path, overwrite = false)
   dest = {}
   File.open(path).each_line do |line|
-    line.strip!
-    if /^(domain|class)\s*=\s*(\S+)/.match(line)
+    case line
+    when /^(domain|class)\s*=\s*(\S+)/
       dest[$1.to_sym] = $2
-    elsif m = /^(?:trackers|hosts)\s*=\s*(.*)/.match(line)
+    when /^(?:trackers|hosts)\s*=\s*(.*)/
       dest[:hosts] = $1.split(/\s*,\s*/)
-    elsif m = /^timeout\s*=\s*(.*)/.match(line)
-      dest[:timeout] = m[1].to_f
+    when /^timeout\s*=\s*(.*)/
+      dest[:timeout] = $1.to_f
+    when /^noclobber\s*=\s*true\s*/
+      dest[:noclobber] = true
     else
-      STDERR.puts "Ignored configuration line: #{line}" unless /^#/.match(line)
+      warn "Ignored configuration line: #{line}" unless /^#/.match(line)
     end
   end
   dest
@@ -33,6 +42,7 @@ cli_cfg = {}
 config_file = nil
 ls_l = false
 ls_h = false
+chunk = false
 test = {}
 cat = { :raw => false }
 
@@ -50,15 +60,19 @@ ARGV.options do |x|
 
   x.on('-e', 'True if key exists') { test[:e] = true }
   x.on('-r', '--raw', 'show raw big_info file information') { cat[:raw] = true }
+  x.on('-n', '--no-clobber', 'do not clobber existing key') do
+    cli_cfg[:noclobber] = true
+  end
 
   x.on('-C', '--class=s', 'class') { |klass| cli_cfg[:class] = klass }
   x.on('-d', '--domain=s', 'domain') { |domain| cli_cfg[:domain] = domain }
   x.on('-l', "long listing format (`ls' command)") { ls_l = true }
   x.on('-h', '--human-readable',
        "print sizes in human-readable format (`ls' command)") { ls_h = true }
-
+  x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
   x.separator ''
   x.on('--help', 'Show this help message.') { puts x; exit }
+  x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
   x.parse!
 end
 
@@ -86,18 +100,17 @@ err = []
 err << "trackers must be specified" if cfg[:hosts].nil? || cfg[:hosts].empty?
 err << "domain must be specified" unless cfg[:domain]
 if err.any?
-  STDERR.puts "Errors:\n  #{err.join("\n  ")}"
-  STDERR.puts ARGV.options
+  warn "Errors:\n  #{err.join("\n  ")}"
+  warn ARGV.options
   exit 1
 end
 
 unless cmd = ARGV.shift
-  STDERR.puts ARGV.options
+  warn ARGV.options
   exit 1
 end
 
 cfg[:timeout] ||= 30 # longer timeout for interactive use
-include MogileFS::Util
 mg = MogileFS::MogileFS.new(cfg)
 
 def store_file_retry(mg, key, storage_class, filepath)
@@ -107,56 +120,57 @@ def store_file_retry(mg, key, storage_class, filepath)
   rescue MogileFS::UnreadableSocketError,
          MogileFS::Backend::NoDevicesError => err
     if ((tries += 1) < 10)
-      STDERR.puts "Retrying on error: #{err}: #{err.message} tries: #{tries}"
+      warn "Retrying on error: #{err}: #{err.message} tries: #{tries}"
       retry
     else
-      STDERR.puts "FATAL: #{err}: #{err.message} tries: #{tries}"
+      warn "FATAL: #{err}: #{err.message} tries: #{tries}"
     end
     exit 1
   end
 end
 
+def human_size(size)
+  suff = ''
+  %w(K M G).each do |s|
+    size /= 1024.0
+    if size <= 1024
+      suff = s
+      break
+    end
+  end
+  sprintf("%.1f%s", size, suff)
+end
+
 begin
   case cmd
   when 'cp'
     filename = ARGV.shift or raise ArgumentError, '<filename> <key>'
-    key = ARGV.shift or raise ArgumentError, '<filename> <key>'
+    dkey = ARGV.shift or raise ArgumentError, '<filename> <key>'
     ARGV.shift and raise ArgumentError, '<filename> <key>'
-    store_file_retry(mg, key, cfg[:class], filename)
+    cfg[:noclobber] && mg.exist?(dkey) and
+      abort "`#{dkey}' already exists and -n/--no-clobber was specified"
+    store_file_retry(mg, dkey, cfg[:class], filename)
   when 'cat'
     ARGV.empty? and raise ArgumentError, '<key1> [<key2> ...]'
     ARGV.each do |key|
       if (!cat[:raw] && key =~ /^_big_info:/)
-        mg.bigfile_write(key, STDOUT, {:verify => true})
+        mg.bigfile_write(key, $stdout, {:verify => true})
       else
-        mg.get_file_data(key) { |fp| sysrwloop(fp, STDOUT) }
+        mg.get_file_data(key, $stdout)
       end
     end
   when 'ls'
     prefixes = ARGV.empty? ? [ nil ] : ARGV
-    prefixes.each do |prefix|
-      mg.each_key(prefix) do |key|
-        if ls_l
-          path_nr = "% 2d" % mg.get_paths(key).size
-          size = mg.size(key)
-          if ls_h && size > 1024
-            suff = ''
-            %w(K M G).each do |s|
-              size /= 1024.0
-              suff = s
-              break if size <= 1024
-            end
-            size = sprintf("%.1f%s", size, suff)
-          else
-            size = size.to_s
-          end
-          size = (' ' * (12 - size.length)) << size # right justify
-          puts [ path_nr, size, key ].pack("A4 A16 A32")
-        else
-          puts key
-        end
+    if ls_l
+      each_key = lambda do |key, size, devcount|
+        size = ls_h && size > 1024 ? human_size(size) : size.to_s
+        size = (' ' * (12 - size.length)) << size # right justify
+        puts [ sprintf("% 2d", devcount), size, key ].pack("A4 A16 A*")
       end
+    else
+      each_key = lambda { |key| puts key }
     end
+    prefixes.each { |prefix| mg.each_key(prefix, &each_key) }
   when 'rm'
     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
     ARGV.each { |key| mg.delete(key) }
@@ -167,35 +181,67 @@ begin
     mg.rename(from, to)
   when 'stat' # this outputs a RFC822-like format
     ARGV.empty? and raise ArgumentError, '<key1> [<key2>]'
-    ARGV.each_with_index do |key, i|
-      if size = mg.size(key)
+    ok = true
+    ARGV.each_with_index do |key,j|
+      begin
+        info = mg.file_info(key)
         puts "Key: #{key}"
-        puts "Size: #{size}"
-        mg.get_paths(key).each_with_index do |path,i|
+        puts "Size: #{info['length']}"
+        puts "Class: #{info['class']}"
+        o = { :pathcount => info["devcount"] }
+        mg.get_paths(key, o).each_with_index do |path,i|
           puts "URL-#{i}: #{path}"
         end
-        puts ""
-      else
-        STDERR.puts "No such key: #{key}"
+        puts "" if ARGV.size != (j + 1)
+      rescue MogileFS::Backend::UnknownKeyError
+        warn "No such key: #{key}"
+        ok = false
       end
     end
+    exit(ok)
   when 'tee'
     require 'tempfile'
-    key = ARGV.shift or raise ArgumentError, '<key>'
+    dkey = ARGV.shift or raise ArgumentError, '<key>'
     ARGV.shift and raise ArgumentError, '<key>'
-    cfg[:class] or raise ArgumentError, 'E: --class must be specified'
-    buf = ''
-    tmp = Tempfile.new('mog-tee') # TODO: explore Transfer-Encoding:chunked :)
-
-    # if stdout is pointing to /dev/null, don't bother installing the filter.
-    STDOUT.sync = true
-    tee_filter = File.stat('/dev/null') == STDOUT.stat ?
-                 nil : Proc.new { |buf| STDOUT.write(buf); buf }
-    begin
-      sysrwloop(STDIN, tmp, tee_filter)
-      store_file_retry(mg, key, cfg[:class], tmp.path)
-    ensure
-      tmp.close!
+    cfg[:noclobber] && mg.exist?(dkey) and
+      abort "`#{dkey}' already exists and -n/--no-clobber was specified"
+    skip_tee = File.stat('/dev/null') == $stdout.stat
+
+    if chunk
+      if skip_tee
+        tee_obj = $stdin
+      else
+        tee_obj = lambda do |*args|
+          buf = $stdin.readpartial(*args)
+          $stdout.write(buf)
+          buf
+        end
+        class << tee_obj
+          alias readpartial call
+        end
+      end
+      mg.store_file(dkey, cfg[:class], tee_obj)
+    else # buffer input, first
+      tmp = Tempfile.new('mog-tee')
+      tmp.sync = true
+
+      # if stdout is pointing to /dev/null, don't bother installing the filter.
+      tee_obj = tmp
+      unless skip_tee
+        tee_obj = lambda do |buf|
+          $stdout.write(buf)
+          tmp.write(buf)
+        end
+        class << tee_obj
+          alias write call
+        end
+      end
+      begin
+        MogileFS.io.copy_stream($stdin, tee_obj)
+        store_file_retry(mg, dkey, cfg[:class], tmp.path)
+      ensure
+        tmp.close!
+      end
     end
   when 'test'
     truth, ok = true, nil
@@ -213,20 +259,15 @@ begin
       raise ArgumentError, "Too many arguments"
     end
 
-    paths = mg.get_paths(key)
-    if test[:e]
-      ok = !!(paths && paths.size > 0)
-    else
-      raise ArgumentError, "Unknown flag: -#{test.keys.first}"
-    end
-
+    test[:e] or raise ArgumentError, "Unknown flag: -#{test.keys.first}"
+    ok = mg.exist?(key)
     truth or ok = ! ok
     exit ok ? 0 : 1
   else
     raise ArgumentError, "Unknown command: #{cmd}"
   end
 rescue ArgumentError => err
-  STDERR.puts "Usage: #{$0} #{cmd} #{err.message}"
+  warn "Usage: #{$0} #{cmd} #{err.message}"
   exit 1
 end
 exit 0