mogilefs-client 3.0.0 → 3.1.0

data/.document

@@ -1,3 +1,4 @@
+NEWS
 History
 ChangeLog
 LICENSE
@@ -9,3 +10,5 @@ lib/mogilefs/admin.rb
 lib/mogilefs/backend.rb
 lib/mogilefs/client.rb
 lib/mogilefs/mogilefs.rb
+lib/mogilefs/new_file.rb
+lib/mogilefs/new_file/writer.rb

data/.gitignore

@@ -6,3 +6,4 @@
 /NEWS
 /LATEST
 /lib/mogilefs/version.rb
+/Manifest.txt

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.0.0"
+DEF_VER = "v3.1.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/GNUmakefile

@@ -34,15 +34,7 @@ $(T):
 	$(run_test)
 
 RUBY_VERSION_FILE = lib/mogilefs/version.rb
-# using make instead of rake since Rakefile takes too long to load
-manifest: Manifest.txt
-Manifest.txt:
-	git ls-files > $@+
-	echo $(RUBY_VERSION_FILE) >> $@+
-	cmp $@+ $@ || mv $@+ $@
-	$(RM) -f $@+
-
-package: manifest
+package:
 	git diff --exit-code HEAD^0
 	$(RM) -r pkg/
 	rake fix_perms
@@ -55,7 +47,7 @@ flay: $(libs)
 	flay $(flay_flags) $^
 flog: $(libs)
 	flog $(flog_flags) $^
-.PHONY: $(T) Manifest.txt
+.PHONY: $(T)
 
 check-warnings:
 	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \

data/History

@@ -1,3 +1,6 @@
+This document is for historical releases.  See the auto-generated
+NEWS document for the latest releases.
+
 = 2.2.0
 * internal cleanups (no public API breakage)
 * refactor backend socket/connection handling for reliability

data/Rakefile

@@ -2,10 +2,18 @@ require 'rubygems'
 require 'hoe'
 load "./GIT-VERSION-GEN"
 
+include Rake::DSL if defined?(Rake::DSL)
 $:.unshift 'lib'
 require 'mogilefs'
 Hoe.plugin :seattlerb
 
+manifest = "Manifest.txt"
+if ! File.exist?(manifest) ||
+   File.stat(manifest).mtime < File.stat(RVF).mtime
+  system("git ls-files > #{manifest}")
+  File.open(manifest, "a") { |fp| fp.puts("lib/mogilefs/version.rb") }
+end
+
 Hoe.spec 'mogilefs-client' do
   self.rubyforge_name = 'seattlerb'
   developer 'Eric Wong', 'normalperson@yhbt.net'

data/bin/mog

@@ -14,8 +14,7 @@ if md5_trailer_nodes = ENV["MD5_TRAILER_NODES"]
 end
 
 # this is to be compatible with config files used by the Perl tools
-def parse_config_file!(path, overwrite = false)
-  dest = {}
+def parse_config_file!(path, dest = {})
   File.open(path).each_line do |line|
     case line
     when /^(domain|class)\s*=\s*(\S+)/
@@ -43,6 +42,7 @@ config_file = nil
 ls_l = false
 ls_h = false
 chunk = false
+range = false
 test = {}
 cat = { :raw => false }
 
@@ -70,6 +70,7 @@ ARGV.options do |x|
   x.on('-h', '--human-readable',
        "print sizes in human-readable format (`ls' command)") { ls_h = true }
   x.on('--chunk', "chunk uploads (`tee' command)") { chunk = true }
+  x.on('--range', "stream partial uploads (`tee' command)") { range = true }
   x.separator ''
   x.on('--help', 'Show this help message.') { puts x; exit }
   x.on('--version', 'Show --version') { puts "#$0 #{MogileFS::VERSION}"; exit }
@@ -77,7 +78,7 @@ ARGV.options do |x|
 end
 
 # parse the config file specified at the command-line
-file_cfg = config_file ? parse_config_file!(config_file, true) : {}
+file_cfg = config_file ? parse_config_file!(config_file) : {}
 
 # read environment variables, too.  This Ruby API favors the term
 # "hosts", however upstream MogileFS teminology favors "trackers" instead.
@@ -188,6 +189,7 @@ begin
         puts "Key: #{key}"
         puts "Size: #{info['length']}"
         puts "Class: #{info['class']}"
+        checksum = info['checksum'] and puts "Checksum: #{checksum}"
         o = { :pathcount => info["devcount"] }
         mg.get_paths(key, o).each_with_index do |path,i|
           puts "URL-#{i}: #{path}"
@@ -200,49 +202,27 @@ begin
     end
     exit(ok)
   when 'tee'
-    require 'tempfile'
+    abort "--range and --chunk are incompatible" if range && chunk
     dkey = ARGV.shift or raise ArgumentError, '<key>'
     ARGV.shift and raise ArgumentError, '<key>'
     cfg[:noclobber] && mg.exist?(dkey) and
       abort "`#{dkey}' already exists and -n/--no-clobber was specified"
     skip_tee = File.stat('/dev/null') == $stdout.stat
+    largefile = :tempfile
+    largefile = :content_range if range
+    largefile = :stream if chunk
 
-    if chunk
-      if skip_tee
-        tee_obj = $stdin
-      else
-        tee_obj = lambda do |*args|
-          buf = $stdin.readpartial(*args)
-          $stdout.write(buf)
-          buf
-        end
-        class << tee_obj
-          alias readpartial call
-        end
-      end
-      mg.store_file(dkey, cfg[:class], tee_obj)
-    else # buffer input, first
-      tmp = Tempfile.new('mog-tee')
-      tmp.sync = true
-
-      # if stdout is pointing to /dev/null, don't bother installing the filter.
-      tee_obj = tmp
-      unless skip_tee
-        tee_obj = lambda do |buf|
-          $stdout.write(buf)
-          tmp.write(buf)
-        end
-        class << tee_obj
-          alias write call
-        end
-      end
+    io = mg.new_file(dkey, :class => cfg[:class], :largefile => largefile)
+    begin
+      buf = $stdin.readpartial(16384)
       begin
-        MogileFS.io.copy_stream($stdin, tee_obj)
-        store_file_retry(mg, dkey, cfg[:class], tmp.path)
-      ensure
-        tmp.close!
-      end
+        io.write(buf)
+        $stdout.write(buf) unless skip_tee
+        $stdin.readpartial(16384, buf)
+      end while true
+    rescue EOFError
     end
+    io.close
   when 'test'
     truth, ok = true, nil
     raise ArgumentError, "-e must be specified" unless (test.size == 1)

data/examples/stale_fid_checker.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+# This requires:
+# * net-http-persistent RubyGem
+# * Ruby 1.9.2+
+# * upstream MogileFS::Server 2.45 or later
+$stdout.sync = $stderr.sync = true
+usage = <<EOF
+Usage: #$0 -t TRACKERS"
+
+The output of this script can be piped to awk + curl to DELETE the files:
+#$0 -t TRACKERS | awk '{system("curl -XDELETE "$3)}'
+EOF
+
+require 'uri'
+require 'optparse'
+require 'mogilefs'
+require 'net/http/persistent'
+Thread.abort_on_exception = true
+MogileFS::VERSION <= "3.0.0" and
+  abort "Upgrade mogilefs-client (to a version that distributes this script)" \
+        "MogileFS::Admin#each_fid is probably broken in this version"
+
+trackers = []
+ARGV.options do |x|
+  x.banner = usage.strip
+  x.separator ''
+  x.on('-t', '--trackers=host1[,host2]', '--hosts=host1[,host2]',
+       Array, 'hostnames/IP addresses of trackers') do |args|
+    trackers = args
+  end
+
+  x.on('-h', '--help', 'Show this help message.') { puts x; exit }
+  x.parse!
+end
+
+adm = MogileFS::Admin.new(:hosts => trackers)
+NHP = Net::HTTP::Persistent.new(File.basename($0))
+client = MogileFS::MogileFS.new(:hosts => trackers, :domain => "none")
+
+def start_perdev_thread(pfx)
+  todo = Queue.new
+  done = Queue.new
+  Thread.new do
+    while fid_path = todo.shift
+      path = "#{pfx}#{fid_path}"
+      uri = URI.parse(path)
+      begin
+        resp = NHP.request(uri, Net::HTTP::Head.new(uri.path))
+        done << [ path, resp ]
+      rescue => err
+        done << [ path, err ]
+      end
+    end
+  end
+  [ todo, done ]
+end
+
+def setup_devices(dev_map, adm)
+  hosts = {}
+  adm.get_hosts.each do |host|
+    hosts[host["hostid"]] = "http://#{host['hostip']}:#{host['http_port']}/"
+  end
+
+  adm.get_devices.each do |device|
+    pfx = hosts[device["hostid"]] + "dev#{device['devid']}"
+    todo, done = start_perdev_thread(pfx)
+    dev_map[todo] = done
+  end
+end
+
+def check(bad, curfid, rv)
+  path, resp = rv
+  case resp
+  when Net::HTTPNotFound # good
+  when Net::HTTPOK
+    bad << "#{curfid} #{resp.content_length} #{path}\n"
+  else
+    warn "E: #{resp.inspect} (#{resp.class}) #{path}"
+  end
+end
+
+dev_map = {}
+setup_devices(dev_map, adm)
+next_fid = 0
+adm.each_fid do |fid|
+  fidid = fid["fid"]
+
+  if fidid != next_fid
+    (next_fid..(fidid - 1)).each do |curfid|
+      nfid = sprintf("%010u", curfid)
+      /\A(\d)(\d{3})(\d{3})(?:\d{3})\z/ =~ nfid
+      fid_path = "/#$1/#$2/#$3/#{nfid}.fid"
+      bad = []
+      dev_map.each_key { |todo| todo << fid_path }
+      dev_map.each_value { |done| check(bad, curfid, done.shift) }
+      next if bad.empty?
+
+      begin
+        info = client.file_debug(curfid)
+        abort "BUG: #{info.inspect} found!" if info["fid_dkey"]
+      rescue MogileFS::Backend::UnknownFidError
+      end
+
+      puts bad.join
+    end
+  end
+  next_fid = fidid + 1
+end

data/lib/mogilefs.rb

@@ -21,7 +21,9 @@ module MogileFS
 
   # Raised when a backend is in read-only mode
   class ReadOnlyError < Error
-    def message; 'readonly mogilefs'; end
+    def message # :nodoc:
+      'readonly mogilefs'
+    end
   end
 
   # Raised when an upload is impossible

data/lib/mogilefs/admin.rb

@@ -12,7 +12,7 @@ class MogileFS::Admin < MogileFS::Client
     low = -1
     rv = 0
     begin
-      fids = list_fids(low + 1)
+      fids = list_fids(low)
       fids.each { |fid| yield fid }
       rv += fids.size
     end while last = fids[-1] and low = last["fid"]
@@ -273,6 +273,7 @@ class MogileFS::Admin < MogileFS::Client
     rv
   end
 
+  # Clears the tracker caches.  Not implemented in all versions of MogileFS
   def clear_cache
     @backend.clear_cache
   end

data/lib/mogilefs/backend.rb

@@ -42,6 +42,14 @@ class MogileFS::Backend
     BACKEND_ERRORS[err_snake] = const_get(err_camel)
   end
 
+  def self.const_missing(name) # :nodoc:
+    if /Error\z/ =~ name.to_s
+      const_set(name, Class.new(MogileFS::Error))
+    else
+      super name
+    end
+  end
+
   ##
   # The last error
 
@@ -62,6 +70,7 @@ class MogileFS::Backend
 
   def initialize(args)
     @hosts = args[:hosts]
+    @fail_timeout = args[:fail_timeout] || 5
     raise ArgumentError, "must specify at least one host" unless @hosts
     raise ArgumentError, "must specify at least one host" if @hosts.empty?
     unless @hosts == @hosts.select { |h| h =~ /:\d+$/ } then
@@ -90,6 +99,7 @@ class MogileFS::Backend
   add_command :create_open
   add_command :create_close
   add_idempotent_command :get_paths
+  add_idempotent_command :noop
   add_command :delete
   add_idempotent_command :sleep
   add_command :rename
@@ -115,44 +125,6 @@ class MogileFS::Backend
   add_command :set_state
   add_command :replicate_now
 
-  # Errors copied from MogileFS/Worker/Query.pm
-  add_error 'dup'
-  add_error 'after_mismatch'
-  add_error 'bad_params'
-  add_error 'class_exists'
-  add_error 'class_has_files'
-  add_error 'class_not_found'
-  add_error 'db'
-  add_error 'domain_has_files'
-  add_error 'domain_exists'
-  add_error 'domain_not_empty'
-  add_error 'domain_not_found'
-  add_error 'failure'
-  add_error 'host_exists'
-  add_error 'host_mismatch'
-  add_error 'host_not_empty'
-  add_error 'host_not_found'
-  add_error 'invalid_chars'
-  add_error 'invalid_checker_level'
-  add_error 'invalid_mindevcount'
-  add_error 'key_exists'
-  add_error 'no_class'
-  add_error 'no_devices'
-  add_error 'no_domain'
-  add_error 'no_host'
-  add_error 'no_ip'
-  add_error 'no_key'
-  add_error 'no_port'
-  add_error 'none_match'
-  add_error 'plugin_aborted'
-  add_error 'state_too_high'
-  add_error 'size_verify_error'
-  add_error 'unknown_command'
-  add_error 'unknown_host'
-  add_error 'unknown_key'
-  add_error 'unknown_state'
-  add_error 'unreg_domain'
-
   def shutdown_unlocked(do_raise = false) # :nodoc:
     @pending = []
     if @socket
@@ -254,11 +226,13 @@ class MogileFS::Backend
 
   # Performs the +cmd+ request with +args+.
   def do_request(cmd, args, idempotent = false)
-    request = make_request cmd, args
+    no_raise = args.delete(:ruby_no_raise)
+    request = make_request(cmd, args)
     @mutex.synchronize do
       begin
         io = dispatch_unlocked(request)
-        line = io.timed_gets(@timeout) and return parse_response(line)
+        line = io.timed_gets(@timeout) and
+          return parse_response(line, no_raise ? request : nil)
 
         idempotent or
           raise EOFError, "end of file reached after: #{request.inspect}"
@@ -349,17 +323,15 @@ class MogileFS::Backend
   def socket
     return @socket if @socket and not @socket.closed?
 
-    now = Time.now
-
     @hosts.shuffle.each do |host|
-      next if @dead.include?(host) and @dead[host][0] > now - 5
+      next if dead = @dead[host] and dead[0] > (Time.now - @fail_timeout)
 
       begin
         addr, port = host.split(/:/)
         @socket = MogileFS::Socket.tcp(addr, port, @timeout)
         @active_host = host
       rescue SystemCallError, MogileFS::Timeout => err
-        @dead[host] = [ now, err ]
+        @dead[host] = [ Time.now, err ]
         next
       end
 

data/lib/mogilefs/chunker.rb

@@ -1,12 +1,12 @@
 # -*- encoding: binary -*-
-require "digest/md5"
 class MogileFS::Chunker
   CRLF = "\r\n"
   attr_reader :io
 
-  def initialize(io, md5)
+  def initialize(io, md5, expect_md5)
     @io = io
-    @md5 = md5 ? Digest::MD5.new : nil
+    @md5 = md5
+    @expect_md5 = expect_md5
   end
 
   def write(buf)
@@ -20,8 +20,14 @@ class MogileFS::Chunker
 
   def flush
     if @md5
-      content_md5 = [ @md5.digest ].pack('m').strip
-      warn "Content-MD5: #{content_md5}\r\n" if $DEBUG
+      content_md5 = [ @md5.digest ].pack('m').rstrip!
+      if @expect_md5.respond_to?(:call)
+        expect = @expect_md5.call.strip
+        if expect != content_md5
+          raise MogileFS::ChecksumMismatchError,
+            "expected: #{expect.inspect} actual: #{content_md5.inspect}"
+        end
+      end
       @io.write("0\r\nContent-MD5: #{content_md5}\r\n\r\n")
     else
       @io.write("0\r\n\r\n")