mogilefs-client 3.7.1 → 3.8.0

data/test/test_fresh.rb

@@ -197,4 +197,75 @@ class TestMogFresh < Test::Unit::TestCase
     @mogilefsd_pid = nil
     assert_raises(MogileFS::UnreachableBackendError) { client.list_keys }
   end
+
+  def test_admin_setup_new_host_and_devices
+    assert_equal [], @admin.get_hosts
+    args = { :ip => @test_host, :port => @mogstored_http_port }
+    @admin.create_host("me", args)
+    yield_for_monitor_update { @admin.get_hosts.empty? or break }
+    hosts = @admin.get_hosts
+    assert_equal 1, hosts.size
+    host = @admin.get_hosts[0]
+    assert_equal "me", host["hostname"]
+    assert_equal @mogstored_http_port, host["http_port"]
+    assert_nil host["http_get_port"]
+    assert_equal @test_host, host["hostip"]
+    assert_kind_of Integer, host["hostid"]
+    assert_equal hosts, @admin.get_hosts(host["hostid"])
+
+    assert_equal [], @admin.get_devices
+  end
+
+  def test_replicate_now
+    assert_equal({"count" => 0}, @admin.replicate_now)
+  end
+
+  def test_clear_cache
+    assert_nil @admin.clear_cache
+  end
+
+  def test_create_update_delete_class
+    domain = "rbmogtest#{Time.now.strftime('%Y%m%d%H%M%S')}.#{uuid}"
+    @admin.create_domain(domain)
+    yield_for_monitor_update { @admin.get_domains.include?(domain) and break }
+
+    @admin.create_class(domain, "klassy", 1)
+
+    assert_raises(MogileFS::Backend::ClassExistsError) do
+      @admin.create_class(domain, "klassy", 1)
+    end
+
+    @admin.update_class(domain, "klassy",
+                        :mindevcount => 1, :replpolicy => "MultipleHosts(1)")
+
+    tmp = nil
+    yield_for_monitor_update do
+      tmp = @admin.get_domains[domain]["klassy"]
+      break if tmp && tmp["replpolicy"] == "MultipleHosts(1)"
+    end
+    assert tmp, "domain did not show up"
+    assert_equal 1, tmp["mindevcount"]
+    assert_equal "MultipleHosts(1)", tmp["replpolicy"]
+    @admin.update_class(domain, "klassy", 2)
+    ensure
+      @admin.delete_class(domain, "klassy") rescue nil
+  end
+
+  def test_device_file_add
+    add_host_device_domain
+    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
+    r, w = IO.pipe
+    thr = Thread.new do
+      (0..9).each do |i|
+        sleep 0.05
+        w.write("#{i}\n")
+      end
+      w.close
+      :ok
+    end
+    assert_equal 20, client.store_file("pipe", nil, r)
+    assert_equal :ok, thr.value
+    r.close
+    assert_equal "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n", client.get_file_data("pipe")
+  end
 end

data/test/test_mogilefs.rb

@@ -6,10 +6,17 @@ require 'fileutils'
 
 class TestMogileFS__MogileFS < TestMogileFS
   def setup
+    @tmpsrv = []
     @klass = MogileFS::MogileFS
     super
   end
 
+  def tmpsrv(blk)
+    t = TempServer.new(blk)
+    @tmpsrv << t
+    t
+  end
+
   def read_headers(client)
     headers = ""
     while line = client.gets
@@ -40,8 +47,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       client.send("HTTP/1.0 200 OK\r\nContent-Length: 5\r\n\r\ndata!", 0)
       client.close
     end
-    t1 = TempServer.new(svr)
-    t2 = TempServer.new(svr)
+    t1 = tmpsrv(svr)
+    t2 = tmpsrv(svr)
     path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
     path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
 
@@ -49,8 +56,6 @@ class TestMogileFS__MogileFS < TestMogileFS
 
     assert_equal 'data!', @client.get_file_data('key')
     assert_equal 1, accept.stat.size
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_get_file_data_http_not_found_failover
@@ -80,16 +85,14 @@ class TestMogileFS__MogileFS < TestMogileFS
       client.close
     end
 
-    t1 = TempServer.new(svr1)
-    t2 = TempServer.new(svr2)
+    t1 = tmpsrv(svr1)
+    t2 = tmpsrv(svr2)
     path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
     path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
     @backend.get_paths = { 'paths' => 2, 'path1' => path1, 'path2' => path2 }
 
     assert_equal 'data!', @client.get_file_data('key')
     assert_equal 2, accept.stat.size
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_get_file_data_http_block
@@ -117,8 +120,8 @@ class TestMogileFS__MogileFS < TestMogileFS
       client.close
       exit 0
     end
-    t1 = TempServer.new(svr)
-    t2 = TempServer.new(svr)
+    t1 = tmpsrv(svr)
+    t2 = tmpsrv(svr)
     path1 = "http://127.0.0.1:#{t1.port}/dev1/0/000/000/0000000062.fid"
     path2 = "http://127.0.0.1:#{t2.port}/dev2/0/000/000/0000000062.fid"
 
@@ -237,7 +240,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     to_store = Tempfile.new('small')
     to_store.syswrite('data')
 
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       while buf = client.readpartial(666)
         received.syswrite(buf)
@@ -260,14 +263,12 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal("data", a[-1])
     assert_equal("", a[-2])
     assert a.grep(%r{\AContent-Length: 4\z})[0]
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_store_content_http
     received = Tempfile.new('received')
 
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
       seen = ""
@@ -295,8 +296,6 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal("data", a[-1])
     assert_equal("", a[-2])
     assert a.grep(%r{\AContent-Length: 4\z})[0]
-    ensure
-      TempServer.destroy_all!
   end
 
 
@@ -306,7 +305,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     10.times do
       expected += "data"
     end
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
       nr = 0
@@ -343,8 +342,6 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal("data" * 10, a[-1])
     assert_equal("", a[-2])
     assert a.grep(%r{\AContent-Length: 40\z})[0]
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_store_content_multi_dest_failover_path
@@ -355,7 +352,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     received1 = Tempfile.new('received')
     received2 = Tempfile.new('received')
 
-    t1 = TempServer.new(Proc.new do |serv, accept|
+    t1 = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       seen = ""
       while seen !~ /\r\n\r\ndata/
@@ -367,7 +364,7 @@ class TestMogileFS__MogileFS < TestMogileFS
       client.close
     end)
 
-    t2 = TempServer.new(Proc.new do |serv, accept|
+    t2 = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       seen = ""
       while seen !~ /\r\n\r\ndata/
@@ -410,12 +407,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal("", b[-2])
     assert a.grep(%r{\AContent-Length: 4\z})[0]
     assert b.grep(%r{\AContent-Length: 4\z})[0]
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_store_content_http_fail
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
       read_headers(client)
@@ -435,7 +430,7 @@ class TestMogileFS__MogileFS < TestMogileFS
 
   def test_store_content_http_empty
     received = Tempfile.new('received')
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
       received.syswrite(client.recv(4096, 0))
@@ -487,7 +482,7 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal expect_size, to_put.stat.size
 
     readed = Tempfile.new('readed')
-    t = TempServer.new(Proc.new do |serv, accept|
+    t = tmpsrv(Proc.new do |serv, accept|
       client, _ = serv.accept
       client.sync = true
       nr = 0
@@ -523,9 +518,6 @@ class TestMogileFS__MogileFS < TestMogileFS
       assert( system(cmp_bin, expect.path, received.path) )
       break
     end
-
-    ensure
-      TempServer.destroy_all!
   end
 
   def test_store_content_readonly
@@ -868,6 +860,10 @@ class TestMogileFS__MogileFS < TestMogileFS
     assert_equal({}, @client.sleep(2))
   end
 
+  def teardown
+    @tmpsrv.each { |t| t.destroy! }
+  end
+
   private
 
     # tested with 1000, though it takes a while

data/test/test_mogilefs_integration.rb

@@ -1,12 +1,16 @@
 # -*- encoding: binary -*-
-require './test/integration'
+require './test/fresh'
 
-class TestMogileFSIntegration < TestMogIntegration
+class TestMogileFSIntegration < Test::Unit::TestCase
+  include TestFreshSetup
   def setup
-    super
+    setup_mogilefs
+    add_host_device_domain
     @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
   end
 
+  alias teardown teardown_mogilefs
+
   def test_CRUD
     assert ! @client.exist?("CRUD")
     assert_equal 4, @client.store_content("CRUD", "default", "DATA")
@@ -157,9 +161,14 @@ class TestMogileFSIntegration < TestMogIntegration
     end
   end
 
-  # TODO: move this to a fresh instance
   def test_admin_each_fid
     admin = MogileFS::Admin.new(:hosts => @trackers)
+    input = %w(a b c d e)
+    input.each do |k|
+      rv = @client.new_file(k)
+      rv.write(k)
+      assert_nil rv.close
+    end
     seen = {}
     count = admin.each_fid do |info|
       seen[info["fid"]] = true
@@ -171,7 +180,8 @@ class TestMogileFSIntegration < TestMogIntegration
       assert_kind_of String, info["domain"]
     end
     assert_equal count, seen.size
-  end if ENV["TEST_EXPENSIVE"]
+    assert_equal count, input.size
+  end
 
   def test_new_file_no_block
     rv = @client.new_file("no_block")

data/test/test_mogilefs_integration_large_pipe.rb

@@ -1,13 +1,17 @@
 # -*- encoding: binary -*-
-require './test/integration'
+require './test/fresh'
 require "digest/sha1"
 
-class TestMogileFSLargePipe< TestMogIntegration
+class TestMogileFSLargePipe < Test::Unit::TestCase
+  include TestFreshSetup
   def setup
-    super
+    setup_mogilefs
+    add_host_device_domain
     @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
   end
 
+  alias teardown teardown_mogilefs
+
   def test_large_pipe_test
     junk = File.open("/dev/urandom") { |fp| fp.read(1024) }
     junk *= 32

data/test/test_mogilefs_integration_list_keys.rb

@@ -1,13 +1,17 @@
-
 # -*- encoding: binary -*-
-require './test/integration'
+require './test/fresh'
+
+class TestMogileFSIntegrationListKeys < Test::Unit::TestCase
+  include TestFreshSetup
 
-class TestMogileFSIntegrationListKeys < TestMogIntegration
   def setup
-    super
+    setup_mogilefs
+    add_host_device_domain
     @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
   end
 
+  alias teardown teardown_mogilefs
+
   def test_list_keys
     k = %w(a b c d e f g)
     k.each { |x| @client.store_content("lk_#{x}", nil, x) }

data/test/test_mogtool_bigfile.rb

@@ -1,5 +1,5 @@
 # -*- encoding: binary -*-
-require "./test/integration"
+require "./test/fresh"
 require "net/http"
 ok = true
 unless File.executable?(`which mogtool 2>/dev/null`.strip)
@@ -7,7 +7,8 @@ unless File.executable?(`which mogtool 2>/dev/null`.strip)
   ok = false
 end
 
-class TestMogtoolBigfile < TestMogIntegration
+class TestMogtoolBigfile < Test::Unit::TestCase
+  include TestFreshSetup
   buf = File.open("/dev/urandom") { |fp| fp.read(1024) }
   buf *= 1024
   RAND = Tempfile.new("rand")
@@ -18,12 +19,15 @@ class TestMogtoolBigfile < TestMogIntegration
   RAND_SHA1 = sha1.hexdigest
 
   def setup
-    super
+    setup_mogilefs
+    add_host_device_domain
     RAND.rewind
     @big_uuid = "big-#{uuid}"
     @client = MogileFS::MogileFS.new(:hosts => @trackers, :domain => @domain)
   end
 
+  alias teardown teardown_mogilefs
+
   def mogtool!(*args)
     x!("mogtool", "--trackers=#{@trackers.join(',')}",
        "--domain=#@domain", *args)

metadata

@@ -1,72 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: mogilefs-client
 version: !ruby/object:Gem::Version
-  version: 3.7.1
-  prerelease: 
+  version: 3.8.0
 platform: ruby
 authors:
-- Eric Wong
+- mogilefs-client hackers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-10 00:00:00.000000000 Z
+date: 2015-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: olddoc
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.10'
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.10'
-- !ruby/object:Gem::Dependency
-  name: hoe
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '3.5'
-description: ! '["A MogileFS client library for Ruby.  MogileFS is an open source\ndistributed
-  filesystem, see: http://mogilefs.org for more details.  This\nlibrary allows any
-  Ruby application to read, write and delete files in a\nMogileFS instance."]'
-email:
-- normalperson@yhbt.net
+        version: '1.0'
+description: |-
+  A MogileFS client library for Ruby.  MogileFS is an open source
+  distributed filesystem, see: http://mogilefs.org for more details.  This
+  library allows any Ruby application to read, write and delete files in a
+  MogileFS instance.
+email: e@80x24.org
 executables:
 - mog
 extensions: []
-extra_rdoc_files:
-- Manifest.txt
+extra_rdoc_files: []
 files:
-- .document
-- .gitignore
-- .wrongdoc.yml
+- ".document"
+- ".gitignore"
+- ".manifest"
+- ".olddoc.yml"
+- GIT-VERSION-FILE
 - GIT-VERSION-GEN
 - GNUmakefile
 - HACKING
 - History
+- LATEST
 - LICENSE
+- NEWS
 - README
-- Rakefile
 - TODO
 - bin/mog
+- examples/mog-sync.rb
 - examples/stale_fid_checker.rb
+- examples/usage_fetcher.rb
 - lib/mogilefs.rb
 - lib/mogilefs/admin.rb
 - lib/mogilefs/backend.rb
@@ -93,12 +79,14 @@ files:
 - lib/mogilefs/socket/pure_ruby.rb
 - lib/mogilefs/socket_common.rb
 - lib/mogilefs/util.rb
+- lib/mogilefs/version.rb
+- mogilefs-client.gemspec
+- pkg.mk
 - setup.rb
 - test/.gitignore
 - test/aggregate.rb
 - test/exec.rb
 - test/fresh.rb
-- test/integration.rb
 - test/setup.rb
 - test/socket_test.rb
 - test/test_admin.rb
@@ -118,53 +106,28 @@ files:
 - test/test_mogtool_bigfile.rb
 - test/test_mysql.rb
 - test/test_pool.rb
-- Manifest.txt
-- ChangeLog
-- NEWS
-- lib/mogilefs/version.rb
-- .gemtest
-homepage: http://bogomips.org/mogilefs-client/
-licenses: []
+homepage: http://bogomips.org/mogilefs-client
+licenses:
+- BSD-3-Clause
+metadata: {}
 post_install_message: 
-rdoc_options:
-- --main
-- README
+rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: seattlerb
-rubygems_version: 1.8.23
+rubyforge_project: 
+rubygems_version: 2.4.5
 signing_key: 
-specification_version: 3
-summary: MogileFS client library for Ruby
-test_files:
-- test/test_http_reader.rb
-- test/test_mogilefs_integration_list_keys.rb
-- test/test_db_backend.rb
-- test/test_mogilefs_socket_pure.rb
-- test/test_mysql.rb
-- test/test_backend.rb
-- test/test_mogilefs.rb
-- test/test_mogtool_bigfile.rb
-- test/test_client.rb
-- test/test_admin.rb
-- test/test_mogilefs_socket_kgio.rb
-- test/test_mogilefs_integration.rb
-- test/test_fresh.rb
-- test/test_pool.rb
-- test/test_bigfile.rb
-- test/test_mogilefs_integration_large_pipe.rb
-- test/test_mogstored_rack.rb
-- test/socket_test.rb
+specification_version: 4
+summary: client - MogileFS client library for Ruby
+test_files: []