mogilefs-client 3.3.0 → 3.4.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.3.0"
+DEF_VER = "v3.4.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/lib/mogilefs/admin.rb

@@ -38,8 +38,9 @@ class MogileFS::Admin < MogileFS::Client
 
   def get_hosts(hostid = nil)
     to_i = %w(hostid http_port http_get_port)
-    clean('hosts', 'host',
-          @backend.get_hosts(hostid ? { :hostid => hostid } : {}), true, to_i)
+    want = %w(status hostip hostname altip altmask).concat(to_i)
+    rv = @backend.get_hosts(hostid ? { :hostid => hostid } : {})
+    clean('hosts', 'host', rv, true, to_i, want)
   end
 
   ##
@@ -56,15 +57,26 @@ class MogileFS::Admin < MogileFS::Client
   #     "devid"=>1,
   #     "hostid"=>1,
   #     "mb_used"=>666,
+  #     "utilization"=>0.0,
+  #     "reject_bad_md5"=>false,
+  #     "observed_state"=>"writeable",
   #     "mb_total"=>666666}]
 
   def get_devices(devid = nil)
     to_i = %w(mb_asof mb_free mb_used mb_total devid weight hostid)
+    want = %w(status observed_state).concat(to_i)
     rv = @backend.get_devices(devid ? { :devid => devid } : {})
-    rv = clean('devices', 'dev', rv, true, to_i)
+    rv = clean('devices', 'dev', rv, true, to_i, want)
+    ostates = Hash[%w(readable writeable unreachable).map! { |f| [f,f] }]
+
     rv.each do |row|
       u = row["utilization"] and
         row["utilization"] = nil == u ? nil : u.to_f
+
+      # maps "" to nil (for dead devices)
+      row["observed_state"] = ostates[row["observed_state"]]
+
+      # be sure we do not set this at all for pre-2.60 MogileFS-Server
       case row["reject_bad_md5"]
       when "1"
         row["reject_bad_md5"] = true
@@ -96,9 +108,10 @@ class MogileFS::Admin < MogileFS::Client
 
   def list_fids(from_fid, count = 100)
     to_i = %w(fid devcount length)
+    want = %w(domain class key).concat(to_i)
+    rv = @backend.list_fids(:from => from_fid, :to => count)
     # :to is now :count internally in mogilefsd
-    clean('fid_count', 'fid_',
-          @backend.list_fids(:from => from_fid, :to => count), true, to_i)
+    clean('fid_count', 'fid_', rv, true, to_i, want)
   end
 
   ##
@@ -151,7 +164,8 @@ class MogileFS::Admin < MogileFS::Client
   #     "test" => {
   #       "default" => {
   #         "mindevcount" => 2,
-  #         "replpolicy" => "MultipleHosts()"
+  #         "replpolicy" => "MultipleHosts()",
+  #         "hashtype => nil,
   #       }
   #     }
   #   }
@@ -166,12 +180,13 @@ class MogileFS::Admin < MogileFS::Client
 
     domains = {}
     to_i = %w(mindevcount)
+    want = %w(name replpolicy hashtype mindevcount)
     (1..res['domains'].to_i).each do |i|
-      domain = clean "domain#{i}classes", "domain#{i}class", res, false, to_i
-
-      tmp = domains[res["domain#{i}"].freeze] = {}
+      domain = clean("domain#{i}classes", "domain#{i}class", res, false, to_i,
+                     want)
+      tmp = domains[res["domain#{i}"]] = {}
       domain.each do |d|
-        tmp[d.delete("name").freeze] = d
+        tmp[d.delete("name")] = d
         have_replpolicy ||= d.include?("replpolicy")
       end
     end
@@ -370,23 +385,33 @@ class MogileFS::Admin < MogileFS::Client
   #     "altip"=>"",
   #     "altmask"=>""}]
 
-  def clean(count, prefix, res, underscore = true, to_i = [])
+  def clean(count, prefix, res, underscore = true, to_i = [], want = nil)
     empty = ""
     underscore = underscore ? '_' : empty
 
     # convert array to hash for O(1) lookups
     to_i = to_i.inject({}) { |m,k| m[k] = m }
-
-    keys = res.keys
-    (1..res[count].to_i).map do |i|
-      re = /^#{prefix}#{i}#{underscore}/
-      row = {}
-      keys.grep(re).each do |k|
-        v = res[k]
-        k = k.sub(re, empty).freeze
-        row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
+    if want
+      (1..res[count].to_i).map do |i|
+        row = {}
+        want.each do |k|
+          v = res["#{prefix}#{i}#{underscore}#{k}"] or next
+          row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
+        end
+        row
+      end
+    else
+      keys = res.keys
+      (1..res[count].to_i).map do |i|
+        re = /^#{prefix}#{i}#{underscore}/
+        row = {}
+        keys.grep(re).each do |k|
+          v = res[k]
+          k = k.sub(re, empty)
+          row[k] = to_i.include?(k) ? (empty == v ? nil : v.to_i) : v
+        end
+        row
       end
-      row
     end
   end
 

data/lib/mogilefs/mogilefs.rb

@@ -88,6 +88,57 @@ class MogileFS::MogileFS < MogileFS::Client
     nil
   end
 
+  # Enumerates keys and yields a +file_info+ hash for each key matched by
+  # +prefix+
+  def each_file_info(prefix = "", args = nil)
+    # FIXME: there's a lot of duplicate code from list_keys_verbose here...
+    raise ArgumentError, "need block" unless block_given?
+    ordered = ready = nil
+    on_file_info = lambda do |info|
+      Hash === info or raise info
+      file_info_cleanup(info)
+
+      # deal with trackers with multiple queryworkers responding out-of-order
+      ready[info["key"]] = info
+      while info = ready.delete(ordered[-1])
+        ordered.pop
+        yield info
+      end
+    end
+
+    nr = 0
+    opts = { :domain => @domain }
+    opts[:devices] = 1 if args && args[:devices]
+    after = args ? args[:after] : nil
+    limit = args ? args[:limit] : nil
+
+    begin
+      keys, after = list_keys(prefix, after, limit || 1000)
+      return nr unless keys && keys[0]
+      ordered = keys.reverse
+      ready = {}
+      nr += keys.size
+      limit -= keys.size if limit
+
+      keys.each do |key|
+        opts[:key] = key
+        @backend.pipeline_dispatch(:file_info, opts, &on_file_info)
+      end
+      @backend.pipeline_wait
+    rescue MogileFS::PipelineError, SystemCallError,
+           MogileFS::RequestTruncatedError,
+           MogileFS::UnreadableSocketError,
+           MogileFS::InvalidResponseError, # truncated response
+           MogileFS::Timeout
+      @backend.shutdown
+      keys = (ordered - ready.keys).reverse!
+      retry
+    end while limit == nil || limit > 0
+  rescue
+    @backend.shutdown
+    raise
+  end
+
   # Retrieves the contents of +key+.  If +dst+ is specified, +dst+
   # should be an IO-like object capable of receiving the +write+ method
   # or a path name.  +copy_length+ may be specified to limit the number of
@@ -339,7 +390,7 @@ class MogileFS::MogileFS < MogileFS::Client
 
   def list_keys_verbose(keys, block) # :nodoc:
     # emulate the MogileFS::Mysql interface, slowly...
-    ordered = keys.dup
+    ordered = keys.reverse
     ready = {}
     on_file_info = lambda do |info|
       Hash === info or raise info
@@ -347,8 +398,8 @@ class MogileFS::MogileFS < MogileFS::Client
 
       # deal with trackers with multiple queryworkers responding out-of-order
       ready[info["key"]] = info
-      while info = ready.delete(ordered[0])
-        block.call(ordered.shift, info["length"], info["devcount"])
+      while info = ready.delete(ordered[-1])
+        block.call(ordered.pop, info["length"], info["devcount"])
       end
     end
     opts = { :domain => @domain }
@@ -371,7 +422,7 @@ class MogileFS::MogileFS < MogileFS::Client
            MogileFS::InvalidResponseError, # truncated response
            MogileFS::Timeout
       @backend.shutdown
-      keys = ordered - ready.keys
+      keys = (ordered - ready.keys).reverse!
       retry
     rescue
       @backend.shutdown

data/test/test_mogilefs_integration_list_keys.rb

@@ -36,5 +36,21 @@ class TestMogileFSIntegrationListKeys < TestMogIntegration
       assert_equal "ek_#{n.to_s}", key
       n += 1
     end
+    assert_equal 9, n
+  end
+
+  def test_each_file_info
+    9.times { |i| @client.store_content("ek_#{i}", nil, i.to_s) }
+    n = 0
+    @client.each_file_info do |info|
+      assert_equal @client.domain, info["domain"]
+      assert_equal n.to_s.size, info["length"]
+      assert_kind_of Integer, info["fid"]
+      assert_kind_of Integer, info["devcount"]
+      assert_equal "default", info["class"]
+      assert_equal "ek_#{n}", info["key"]
+      n += 1
+    end
+    assert_equal 9, n
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: mogilefs-client
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 23
   prerelease: 
   segments: 
   - 3
-  - 3
+  - 4
   - 0
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors: 
 - Eric Wong
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-08-11 00:00:00 Z
+date: 2012-10-09 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdoc