mogilefs-client 3.2.0.rc1 → 3.2.0

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.2.0-rc1"
+DEF_VER = "v3.2.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),

data/lib/mogilefs/admin.rb

@@ -37,7 +37,7 @@ class MogileFS::Admin < MogileFS::Client
   #     "altmask"=>""}]
 
   def get_hosts(hostid = nil)
-    to_i = { "hostid" => true, "http_port" => true, "http_get_port" => true }
+    to_i = %w(hostid http_port http_get_port)
     clean('hosts', 'host',
           @backend.get_hosts(hostid ? { :hostid => hostid } : {}), true, to_i)
   end
@@ -59,16 +59,18 @@ class MogileFS::Admin < MogileFS::Client
   #     "mb_total"=>666666}]
 
   def get_devices(devid = nil)
-    to_i = {
-      "mb_asof" => true, "mb_free" => true,
-      "mb_used" => true, "mb_total" => true ,
-      "devid" => true, "weight" => true, "hostid" => true
-    }
+    to_i = %w(mb_asof mb_free mb_used mb_total devid weight hostid)
     rv = @backend.get_devices(devid ? { :devid => devid } : {})
     rv = clean('devices', 'dev', rv, true, to_i)
     rv.each do |row|
       u = row["utilization"] and
         row["utilization"] = nil == u ? nil : u.to_f
+      case row["reject_bad_md5"]
+      when "1"
+        row["reject_bad_md5"] = true
+      when "0"
+        row["reject_bad_md5"] = false
+      end
     end
   end
 
@@ -93,7 +95,7 @@ class MogileFS::Admin < MogileFS::Client
   #     "key"=>"new_new_key"}]
 
   def list_fids(from_fid, count = 100)
-    to_i = { "fid" => true, "devcount" => true, "length" => true }
+    to_i = %w(fid devcount length)
     # :to is now :count internally in mogilefsd
     clean('fid_count', 'fid_',
           @backend.list_fids(:from => from_fid, :to => count), true, to_i)
@@ -163,7 +165,7 @@ class MogileFS::Admin < MogileFS::Client
     have_replpolicy = false
 
     domains = {}
-    to_i = { "mindevcount" => true }
+    to_i = %w(mindevcount)
     (1..res['domains'].to_i).each do |i|
       domain = clean "domain#{i}classes", "domain#{i}class", res, false, to_i
 
@@ -254,6 +256,26 @@ class MogileFS::Admin < MogileFS::Client
     ! @backend.delete_host(:host => host).nil?
   end
 
+  ##
+  # Creates device with Integer +devid+ on +host+
+  # +host+ may be an integer for hostid or String for hostname
+  def create_device(host, devid, opts = {})
+    raise MogileFS::ReadOnlyError if readonly?
+    opts = opts.dup
+
+    case host
+    when Integer
+      opts[:hostid] = host
+    when String
+      opts[:hostname] = host
+    else
+      raise ArgumentError, "host=#{host.inspect} is not a String or Integer"
+    end
+
+    opts[:devid] = devid
+    ! @backend.create_device(opts).nil?
+  end
+
   ##
   # Changes the device status of +device+ on +host+ to +state+ which can be
   # 'alive', 'down', or 'dead'.
@@ -263,6 +285,16 @@ class MogileFS::Admin < MogileFS::Client
     ! @backend.set_state(:host => host, :device => device, :state => state).nil?
   end
 
+  ##
+  # Changes the device weight of +device+ on +host+ to +weight+.
+  # +weight+ should be a non-negative Integer.  Devices with higher
+  # +weight+ values are more likely to be chosen for reads and writes.
+  def change_device_weight(host, device, weight)
+    raise MogileFS::ReadOnlyError if readonly?
+    opts = { :host => host, :device => device, :weight => weight }
+    ! @backend.set_weight(opts).nil?
+  end
+
   # reschedules all deferred replication, returns a hash with the number
   # of files rescheduled:
   #
@@ -330,18 +362,21 @@ class MogileFS::Admin < MogileFS::Client
   # Returns:
   #
   #   [{"status"=>"alive",
-  #     "http_get_port"=>"",
-  #     "http_port"=>"",
-  #     "hostid"=>"1",
-  #     "hostip"=>"",
+  #     "http_get_port"=>nil,
+  #     "http_port"=>7600,
+  #     "hostid"=>1,
+  #     "hostip"=>"192.168.1.3",
   #     "hostname"=>"rur-1",
-  #     "remoteroot"=>"/mnt/mogilefs/rur-1",
   #     "altip"=>"",
   #     "altmask"=>""}]
 
   def clean(count, prefix, res, underscore = true, to_i = [])
     empty = ""
     underscore = underscore ? '_' : empty
+
+    # convert array to hash for O(1) lookups
+    to_i = to_i.inject({}) { |m,k| m[k] = m }
+
     keys = res.keys
     (1..res[count].to_i).map do |i|
       re = /^#{prefix}#{i}#{underscore}/

data/lib/mogilefs/backend.rb

@@ -114,6 +114,7 @@ class MogileFS::Backend
   add_idempotent_command :list_fids
   add_idempotent_command :stats
   add_idempotent_command :get_domains
+  add_command :create_device
   add_command :create_domain
   add_command :delete_domain
   add_command :create_class
@@ -124,6 +125,7 @@ class MogileFS::Backend
   add_command :update_host
   add_command :delete_host
   add_command :set_state
+  add_command :set_weight
   add_command :replicate_now
 
   def shutdown_unlocked(do_raise = false) # :nodoc:

data/test/fresh.rb

@@ -124,12 +124,21 @@ EOF
     assert File.directory?("#@docroot/dev2")
     yield_for_monitor_update { @admin.get_hosts.empty? or break }
 
-    # TODO: allow adding devices via our MogileFS::Admin class
-    mogadm!("device", "add", "me", "dev1")
+    me = @admin.get_hosts.find { |x| x["hostname"] == "me" }
+    assert_instance_of Hash, me, me.inspect
+    assert_kind_of Integer, me["hostid"], me
+    assert_equal true, @admin.create_device(me["hostid"], 1)
     yield_for_monitor_update { @admin.get_devices.empty? or break }
     wait_for_usage_file "dev1"
-    mogadm!("device", "add", "me", "dev2")
+    assert_equal true, @admin.create_device("me", 2)
     wait_for_usage_file "dev2"
+
+    # MogileFS::Server 2.60+ shows reject_bad_md5 monitor status
+    dev = @admin.get_devices[0]
+    if dev.include?("reject_bad_md5")
+      assert [true, false].include?(dev["reject_bad_md5"])
+    end
+
     out = err = nil
     tries = 0
     begin

data/test/test_fresh.rb

@@ -6,6 +6,12 @@ class TestMogFresh < Test::Unit::TestCase
   alias setup setup_mogilefs
   alias teardown teardown_mogilefs
 
+  def test_change_device_weight
+    add_host_device_domain
+    assert_equal true, @admin.change_device_weight("me", 1, 50)
+    assert_equal 50, @admin.get_devices(1)[0]["weight"]
+  end
+
   def test_list_keys_invalid_domain
     add_host_device_domain
     domain = @domain + ".non-existent"

metadata

@@ -1,208 +1,154 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: mogilefs-client
-version: !ruby/object:Gem::Version
-  version: !binary |-
-    My4yLjAucmMx
-  prerelease: 6
+version: !ruby/object:Gem::Version 
+  hash: 15
+  prerelease: 
+  segments: 
+  - 3
+  - 2
+  - 0
+  version: 3.2.0
 platform: ruby
-authors:
+authors: 
 - Eric Wong
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-15 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-06-29 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: hoe
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.12'
-  type: :development
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
+    requirements: 
     - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.12'
-description: ! '["A MogileFS client library for Ruby.  MogileFS is an open source\ndistributed
-  filesystem, see: http://mogilefs.org for more details.  This\nlibrary allows any
-  Ruby application to read, write and delete files in a\nMogileFS instance."]'
-email:
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 12
+        version: "2.12"
+  type: :development
+  version_requirements: *id001
+description: |-
+  A MogileFS client library for Ruby.  MogileFS is an open source
+  distributed filesystem, see: http://mogilefs.org for more details.  This
+  library allows any Ruby application to read, write and delete files in a
+  MogileFS instance.
+email: 
 - normalperson@yhbt.net
-executables:
-- !binary |-
-  bW9n
+executables: 
+- mog
 extensions: []
-extra_rdoc_files:
-- !binary |-
-  TWFuaWZlc3QudHh0
-files:
-- !binary |-
-  LmRvY3VtZW50
-- !binary |-
-  LmdpdGlnbm9yZQ==
-- !binary |-
-  Lndyb25nZG9jLnltbA==
-- !binary |-
-  R0lULVZFUlNJT04tR0VO
-- !binary |-
-  R05VbWFrZWZpbGU=
-- !binary |-
-  SEFDS0lORw==
-- !binary |-
-  SGlzdG9yeQ==
-- !binary |-
-  TElDRU5TRQ==
-- !binary |-
-  UkVBRE1F
-- !binary |-
-  UmFrZWZpbGU=
-- !binary |-
-  VE9ETw==
-- !binary |-
-  YmluL21vZw==
-- !binary |-
-  ZXhhbXBsZXMvc3RhbGVfZmlkX2NoZWNrZXIucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL2FkbWluLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL2JhY2tlbmQucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL2JpZ2ZpbGUucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL2JpZ2ZpbGUvZmlsdGVyLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL2NodW5rZXIucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL2NsaWVudC5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL2NvcHlfc3RyZWFtLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL2h0dHBfZmlsZS5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL2h0dHBfcmVhZGVyLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL21vZ2lsZWZzLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL215c3FsLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlL2NvbW1vbi5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlL2NvbnRlbnRfcmFuZ2UucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlL3N0cmVhbS5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlL3RlbXBmaWxlLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL25ld19maWxlL3dyaXRlci5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL3BhdGhzX3NpemUucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL3Bvb2wucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL3NvY2tldC5yYg==
-- !binary |-
-  bGliL21vZ2lsZWZzL3NvY2tldC9rZ2lvLnJi
-- !binary |-
-  bGliL21vZ2lsZWZzL3NvY2tldC9wdXJlX3J1YnkucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL3NvY2tldF9jb21tb24ucmI=
-- !binary |-
-  bGliL21vZ2lsZWZzL3V0aWwucmI=
-- !binary |-
-  c2V0dXAucmI=
-- !binary |-
-  dGVzdC8uZ2l0aWdub3Jl
-- !binary |-
-  dGVzdC9hZ2dyZWdhdGUucmI=
-- !binary |-
-  dGVzdC9leGVjLnJi
-- !binary |-
-  dGVzdC9mcmVzaC5yYg==
-- !binary |-
-  dGVzdC9pbnRlZ3JhdGlvbi5yYg==
-- !binary |-
-  dGVzdC9zZXR1cC5yYg==
-- !binary |-
-  dGVzdC9zb2NrZXRfdGVzdC5yYg==
-- !binary |-
-  dGVzdC90ZXN0X2FkbWluLnJi
-- !binary |-
-  dGVzdC90ZXN0X2JhY2tlbmQucmI=
-- !binary |-
-  dGVzdC90ZXN0X2JpZ2ZpbGUucmI=
-- !binary |-
-  dGVzdC90ZXN0X2NsaWVudC5yYg==
-- !binary |-
-  dGVzdC90ZXN0X2RiX2JhY2tlbmQucmI=
-- !binary |-
-  dGVzdC90ZXN0X2ZyZXNoLnJi
-- !binary |-
-  dGVzdC90ZXN0X2h0dHBfcmVhZGVyLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzX2ludGVncmF0aW9uLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzX2ludGVncmF0aW9uX2xhcmdlX3BpcGUucmI=
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzX2ludGVncmF0aW9uX2xpc3Rfa2V5cy5yYg==
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzX3NvY2tldF9rZ2lvLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ2lsZWZzX3NvY2tldF9wdXJlLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ3N0b3JlZF9yYWNrLnJi
-- !binary |-
-  dGVzdC90ZXN0X21vZ3Rvb2xfYmlnZmlsZS5yYg==
-- !binary |-
-  dGVzdC90ZXN0X215c3FsLnJi
-- !binary |-
-  dGVzdC90ZXN0X3Bvb2wucmI=
-- !binary |-
-  TWFuaWZlc3QudHh0
-- !binary |-
-  Q2hhbmdlTG9n
-- !binary |-
-  TkVXUw==
-- !binary |-
-  bGliL21vZ2lsZWZzL3ZlcnNpb24ucmI=
+
+extra_rdoc_files: 
+- Manifest.txt
+files: 
+- .document
+- .gitignore
+- .wrongdoc.yml
+- GIT-VERSION-GEN
+- GNUmakefile
+- HACKING
+- History
+- LICENSE
+- README
+- Rakefile
+- TODO
+- bin/mog
+- examples/stale_fid_checker.rb
+- lib/mogilefs.rb
+- lib/mogilefs/admin.rb
+- lib/mogilefs/backend.rb
+- lib/mogilefs/bigfile.rb
+- lib/mogilefs/bigfile/filter.rb
+- lib/mogilefs/chunker.rb
+- lib/mogilefs/client.rb
+- lib/mogilefs/copy_stream.rb
+- lib/mogilefs/http_file.rb
+- lib/mogilefs/http_reader.rb
+- lib/mogilefs/mogilefs.rb
+- lib/mogilefs/mysql.rb
+- lib/mogilefs/new_file.rb
+- lib/mogilefs/new_file/common.rb
+- lib/mogilefs/new_file/content_range.rb
+- lib/mogilefs/new_file/stream.rb
+- lib/mogilefs/new_file/tempfile.rb
+- lib/mogilefs/new_file/writer.rb
+- lib/mogilefs/paths_size.rb
+- lib/mogilefs/pool.rb
+- lib/mogilefs/socket.rb
+- lib/mogilefs/socket/kgio.rb
+- lib/mogilefs/socket/pure_ruby.rb
+- lib/mogilefs/socket_common.rb
+- lib/mogilefs/util.rb
+- setup.rb
+- test/.gitignore
+- test/aggregate.rb
+- test/exec.rb
+- test/fresh.rb
+- test/integration.rb
+- test/setup.rb
+- test/socket_test.rb
+- test/test_admin.rb
+- test/test_backend.rb
+- test/test_bigfile.rb
+- test/test_client.rb
+- test/test_db_backend.rb
+- test/test_fresh.rb
+- test/test_http_reader.rb
+- test/test_mogilefs.rb
+- test/test_mogilefs_integration.rb
+- test/test_mogilefs_integration_large_pipe.rb
+- test/test_mogilefs_integration_list_keys.rb
+- test/test_mogilefs_socket_kgio.rb
+- test/test_mogilefs_socket_pure.rb
+- test/test_mogstored_rack.rb
+- test/test_mogtool_bigfile.rb
+- test/test_mysql.rb
+- test/test_pool.rb
+- Manifest.txt
+- ChangeLog
+- NEWS
+- lib/mogilefs/version.rb
 - .gemtest
 homepage: http://bogomips.org/mogilefs-client
 licenses: []
+
 post_install_message: 
-rdoc_options:
+rdoc_options: 
 - --main
 - README
-require_paths:
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>'
-    - !ruby/object:Gem::Version
-      version: 1.3.1
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
+
 rubyforge_project: seattlerb
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: MogileFS client library for Ruby
-test_files:
+test_files: 
 - test/test_http_reader.rb
 - test/test_mogilefs_integration_list_keys.rb
 - test/test_db_backend.rb