mogilefs-client 3.7.1 → 3.8.0

data/examples/usage_fetcher.rb

@@ -0,0 +1,44 @@
+# example to fetch all usage files into the current directory
+# This will download dev*/usage files to the current files with the template:
+#   ${HOSTNAME}_${PORT}_dev$DEVID.txt
+$stderr.sync = $stdout.sync = Thread.abort_on_exception = true
+ARGV.empty? and abort "Usage: #$0 TRACKERS"
+
+require 'net/http/persistent' # gem install net-http-persistent
+require 'uri'
+require 'thread'
+require 'mogilefs'
+queue = Queue.new
+nhp = Net::HTTP::Persistent.new
+adm = MogileFS::Admin.new(hosts: ARGV)
+
+by_hostid = {}
+adm.get_hosts.each { |host| by_hostid[host["hostid"]] = host }
+by_hostid.freeze # by_hostid is read-only at this point
+
+nr = by_hostid.size
+thr = nr.times.map do
+  Thread.new do
+    while dev = queue.shift
+      host = by_hostid[dev["hostid"]]
+      port = host["http_get_port"] || host["http_port"]
+      devid = dev["devid"]
+      url = "http://#{host["hostip"]}:#{port}/dev#{devid}/usage"
+      uri = URI(url)
+      response = nhp.request(uri)
+      body = response.body
+      File.open("#{host["hostname"]}_#{port}_dev#{devid}.txt", "w") do |fp|
+        fp.write(body)
+      end
+    end
+  end
+end
+
+adm.get_devices.each do |dev|
+  case dev["status"]
+  when "alive", "readonly", "drain"
+    queue.push(dev)
+  end
+end
+nr.times { queue.push(nil) } # terminate the threads
+thr.each(&:join)

data/lib/mogilefs/admin.rb

@@ -29,9 +29,9 @@ class MogileFS::Admin < MogileFS::Client
   #
   #   [{"status"=>"alive",
   #     "http_get_port"=>nil,
-  #     "http_port"=>"",
-  #     "hostid"=>"1",
-  #     "hostip"=>"",
+  #     "http_port"=>7500,
+  #     "hostid"=>1,
+  #     "hostip"=>"192.168.1.2",
   #     "hostname"=>"rur-1",
   #     "altip"=>"",
   #     "altmask"=>""}]
@@ -64,17 +64,18 @@ class MogileFS::Admin < MogileFS::Client
 
   def get_devices(devid = nil)
     to_i = %w(mb_asof mb_free mb_used mb_total devid weight hostid)
-    want = %w(status observed_state).concat(to_i)
+    want = %w(status reject_bad_md5 observed_state utilization).concat(to_i)
     rv = @backend.get_devices(devid ? { :devid => devid } : {})
     rv = clean('devices', 'dev', rv, true, to_i, want)
-    ostates = Hash[%w(readable writeable unreachable).map! { |f| [f,f] }]
 
     rv.each do |row|
       u = row["utilization"] and
         row["utilization"] = nil == u ? nil : u.to_f
 
-      # maps "" to nil (for dead devices)
-      row["observed_state"] = ostates[row["observed_state"]]
+      case row["observed_state"]
+      when ""
+        row["observed_state"] = nil
+      end
 
       # be sure we do not set this at all for pre-2.60 MogileFS-Server
       case row["reject_bad_md5"]
@@ -119,6 +120,7 @@ class MogileFS::Admin < MogileFS::Client
   #
   # *** This command no longer works with recent versions of MogileFS ***
   # *** Use mogstats(1) from the MogileFS::Utils package on CPAN ***
+  # *** We will remove this method in 4.x ***
   #
   #   admin.get_stats
   #
@@ -147,9 +149,9 @@ class MogileFS::Admin < MogileFS::Client
       }
     end
 
-    stats.delete 'device' if stats['device'].empty?
-    stats.delete 'file' if stats['file'].empty?
-    stats.delete 'replication' if stats['replication'].empty?
+    %w(device file replication).each do |s|
+      stats.delete(s) if stats[s].empty?
+    end
 
     stats
   end

data/mogilefs-client.gemspec

@@ -0,0 +1,20 @@
+ENV["VERSION"] or abort "VERSION= must be specified"
+require 'olddoc'
+$LOAD_PATH << 'lib'
+require 'mogilefs'
+extend Olddoc::Gemspec
+name, summary, title = readme_metadata
+Gem::Specification.new do |s|
+  manifest = File.read('.manifest').split(/\n/)
+  s.name = 'mogilefs-client'
+  s.version = MogileFS::VERSION
+  s.executables = %w(mog)
+  s.authors = ["#{s.name} hackers"]
+  s.summary = summary
+  s.description = readme_description
+  s.email = Olddoc.config['private_email']
+  s.files = manifest
+  s.add_development_dependency('olddoc', '~> 1.0')
+  s.homepage = Olddoc.config['rdoc_url']
+  s.license = 'BSD-3-Clause'
+end

data/pkg.mk

@@ -0,0 +1,150 @@
+RUBY = ruby
+RAKE = rake
+RSYNC = rsync
+OLDDOC = olddoc
+RDOC = rdoc
+
+GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
+	@./GIT-VERSION-GEN
+-include GIT-VERSION-FILE
+-include local.mk
+DLEXT := $(shell $(RUBY) -rrbconfig -e 'puts RbConfig::CONFIG["DLEXT"]')
+RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
+RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
+lib := lib
+
+ext := $(firstword $(wildcard ext/*))
+ifneq ($(ext),)
+ext_pfx := tmp/ext/$(RUBY_ENGINE)-$(RUBY_VERSION)
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(wildcard $(ext)/*.c $(ext_h))
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_d := $(ext_pfx)/$(ext)/.d
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>> $@
+$(ext_d):
+	@mkdir -p $(@D)
+	@> $@
+$(ext_pfx)/$(ext)/%: $(ext)/% $(ext_d)
+	install -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_d) $(ext_h)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb $(EXTCONF_ARGS)
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	@echo $^ == $@
+	$(MAKE) -C $(@D)
+lib := $(lib):$(ext_pfx)/$(ext)
+build: $(ext_dl)
+else
+build:
+endif
+
+pkg_extra += GIT-VERSION-FILE NEWS LATEST
+NEWS: GIT-VERSION-FILE .olddoc.yml
+	$(OLDDOC) prepare
+LATEST: NEWS
+
+manifest:
+	$(RM) .manifest
+	$(MAKE) .manifest
+
+.manifest: $(pkg_extra)
+	(git ls-files && for i in $@ $(pkg_extra); do echo $$i; done) | \
+		LC_ALL=C sort > $@+
+	cmp $@+ $@ || mv $@+ $@
+	$(RM) $@+
+
+doc:: .document .olddoc.yml $(pkg_extra) $(PLACEHOLDERS)
+	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
+	-find ext -type f -name '*.rbc' -exec rm -f '{}' ';'
+	$(RM) -r doc
+	$(RDOC) -f oldweb
+	$(OLDDOC) merge
+	if test -f COPYING; then install -m644 COPYING doc/COPYING; fi
+	install -m644 NEWS doc/NEWS
+	install -m644 NEWS.atom.xml doc/NEWS.atom.xml
+	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
+
+ifneq ($(VERSION),)
+pkggem := pkg/$(rfpackage)-$(VERSION).gem
+pkgtgz := pkg/$(rfpackage)-$(VERSION).tgz
+
+# ensures we're actually on the tagged $(VERSION), only used for release
+verify:
+	test x"$(shell umask)" = x0022
+	git rev-parse --verify refs/tags/v$(VERSION)^{}
+	git diff-index --quiet HEAD^0
+	test $$(git rev-parse --verify HEAD^0) = \
+	     $$(git rev-parse --verify refs/tags/v$(VERSION)^{})
+
+fix-perms:
+	-git ls-tree -r HEAD | awk '/^100644 / {print $$NF}' | xargs chmod 644
+	-git ls-tree -r HEAD | awk '/^100755 / {print $$NF}' | xargs chmod 755
+
+gem: $(pkggem)
+
+install-gem: $(pkggem)
+	gem install $(CURDIR)/$<
+
+$(pkggem): manifest fix-perms
+	gem build $(rfpackage).gemspec
+	mkdir -p pkg
+	mv $(@F) $@
+
+$(pkgtgz): distdir = $(basename $@)
+$(pkgtgz): HEAD = v$(VERSION)
+$(pkgtgz): manifest fix-perms
+	@test -n "$(distdir)"
+	$(RM) -r $(distdir)
+	mkdir -p $(distdir)
+	tar cf - $$(cat .manifest) | (cd $(distdir) && tar xf -)
+	cd pkg && tar cf - $(basename $(@F)) | gzip -9 > $(@F)+
+	mv $@+ $@
+
+package: $(pkgtgz) $(pkggem)
+
+release:: verify package
+	# push gem to RubyGems.org
+	gem push $(pkggem)
+else
+gem install-gem: GIT-VERSION-FILE
+	$(MAKE) $@ VERSION=$(GIT_VERSION)
+endif
+
+all:: check
+test_units := $(wildcard test/test_*.rb)
+test: check
+check: test-unit
+test-unit: $(test_units)
+$(test_units): build
+	$(RUBY) -I $(lib) $@ $(RUBY_TEST_OPTS)
+
+# this requires GNU coreutils variants
+ifneq ($(RSYNC_DEST),)
+publish_doc:
+	-git set-file-times
+	$(MAKE) doc
+	$(MAKE) doc_gz
+	$(RSYNC) -av doc/ $(RSYNC_DEST)/
+	git ls-files | xargs touch
+endif
+
+# Create gzip variants of the same timestamp as the original so nginx
+# "gzip_static on" can serve the gzipped versions directly.
+doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.gz$$')
+doc_gz:
+	for i in $(docs); do \
+	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
+check-warnings:
+	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
+	  do $(RUBY) -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
+
+ifneq ($(PLACEHOLDERS),)
+$(PLACEHOLDERS):
+	echo olddoc_placeholder > $@
+endif
+
+.PHONY: all .FORCE-GIT-VERSION-FILE doc check test $(test_units) manifest
+.PHONY: check-warnings

data/test/fresh.rb

@@ -7,24 +7,22 @@ require "net/http"
 module TestFreshSetup
   include TestExec
 
-  def setup
-    setup_mogilefs
-  end
-
   def setup_mogilefs(plugins = nil)
+    @teardown_pid = $$
     @test_host = "127.0.0.1"
     setup_mogstored
     @tracker = TCPServer.new(@test_host, 0)
     @tracker_port = @tracker.addr[1]
 
-    @dbname = Tempfile.new(["mogfresh", ".sqlite3"])
-    @mogilefsd_conf = Tempfile.new(["mogilefsd", "conf"])
-    @mogilefsd_pid = Tempfile.new(["mogilefsd", "pid"])
+    @dbname = Tempfile.new(["mogfresh", ".sqlite3"], @docroot)
+    @mogilefsd_conf = Tempfile.new(["mogilefsd", "conf"], @docroot)
+    @mogilefsd_pid = Tempfile.new(["mogilefsd", "pid"], @docroot)
+    @dbpath = @dbname.path
 
-    cmd = %w(mogdbsetup --yes --type=SQLite --dbname) << @dbname.path
+    cmd = %w(mogdbsetup --yes --type=SQLite --dbname) << @dbpath
     x!(*cmd)
 
-    @mogilefsd_conf.puts "db_dsn DBI:SQLite:#{@dbname.path}"
+    @mogilefsd_conf.puts "db_dsn DBI:SQLite:#@dbpath"
     @mogilefsd_conf.write <<EOF
 conf_port #@tracker_port
 listen #@test_host
@@ -63,59 +61,6 @@ EOF
     raise "#@test_host:#{port} never became ready"
   end
 
-  def test_admin_setup_new_host_and_devices
-    assert_equal [], @admin.get_hosts
-    args = { :ip => @test_host, :port => @mogstored_http_port }
-    @admin.create_host("me", args)
-    yield_for_monitor_update { @admin.get_hosts.empty? or break }
-    hosts = @admin.get_hosts
-    assert_equal 1, hosts.size
-    host = @admin.get_hosts[0]
-    assert_equal "me", host["hostname"]
-    assert_equal @mogstored_http_port, host["http_port"]
-    assert_nil host["http_get_port"]
-    assert_equal @test_host, host["hostip"]
-    assert_kind_of Integer, host["hostid"]
-    assert_equal hosts, @admin.get_hosts(host["hostid"])
-
-    assert_equal [], @admin.get_devices
-  end
-
-  def test_replicate_now
-    assert_equal({"count" => 0}, @admin.replicate_now)
-  end
-
-  def test_clear_cache
-    assert_nil @admin.clear_cache
-  end
-
-  def test_create_update_delete_class
-    domain = "rbmogtest#{Time.now.strftime('%Y%m%d%H%M%S')}.#{uuid}"
-    @admin.create_domain(domain)
-    yield_for_monitor_update { @admin.get_domains.include?(domain) and break }
-
-    @admin.create_class(domain, "klassy", 1)
-
-    assert_raises(MogileFS::Backend::ClassExistsError) do
-      @admin.create_class(domain, "klassy", 1)
-    end
-
-    @admin.update_class(domain, "klassy",
-                        :mindevcount => 1, :replpolicy => "MultipleHosts(1)")
-
-    tmp = nil
-    yield_for_monitor_update do
-      tmp = @admin.get_domains[domain]["klassy"]
-      break if tmp && tmp["replpolicy"] == "MultipleHosts(1)"
-    end
-    assert tmp, "domain did not show up"
-    assert_equal 1, tmp["mindevcount"]
-    assert_equal "MultipleHosts(1)", tmp["replpolicy"]
-    @admin.update_class(domain, "klassy", 2)
-    ensure
-      @admin.delete_class(domain, "klassy") rescue nil
-  end
-
   def add_host_device_domain
     assert_equal [], @admin.get_hosts
     args = { :ip => @test_host, :port => @mogstored_http_port }
@@ -161,25 +106,8 @@ EOF
     @domain = domain
   end
 
-  def test_device_file_add
-    add_host_device_domain
-    client = MogileFS::MogileFS.new :hosts => @hosts, :domain => @domain
-    r, w = IO.pipe
-    thr = Thread.new do
-      (0..9).each do |i|
-        sleep 0.05
-        w.write("#{i}\n")
-      end
-      w.close
-      :ok
-    end
-    assert_equal 20, client.store_file("pipe", nil, r)
-    assert_equal :ok, thr.value
-    r.close
-    assert_equal "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\n", client.get_file_data("pipe")
-  end
-
   def teardown_mogilefs
+    return if $$ != @teardown_pid
     if @mogstored_pid
       pid = File.read(@mogstored_pid.path).to_i
       Process.kill(:TERM, pid) if pid > 0
@@ -215,8 +143,8 @@ EOF
     @mogstored_http = TCPServer.new(@test_host, 0)
     @mogstored_mgmt_port = @mogstored_mgmt.addr[1]
     @mogstored_http_port = @mogstored_http.addr[1]
-    @mogstored_conf = Tempfile.new(["mogstored", "conf"])
-    @mogstored_pid = Tempfile.new(["mogstored", "pid"])
+    @mogstored_conf = Tempfile.new(["mogstored", "conf"], @docroot)
+    @mogstored_pid = Tempfile.new(["mogstored", "pid"], @docroot)
     @mogstored_conf.write <<EOF
 pidfile = #{@mogstored_pid.path}
 maxconns = 1000

data/test/setup.rb

@@ -58,13 +58,8 @@ require 'socket'
 class TempServer
   attr_reader :port, :pid
 
-  def self.destroy_all!
-    ObjectSpace.each_object(TempServer) { |t| t.destroy! }
-  end
-
-  at_exit { TempServer.destroy_all! }
-
   def initialize(server_proc, port = nil)
+    @destroy_pid = $$
     @pid = @sock = nil
     @port = port
     retries = 10
@@ -85,6 +80,7 @@ class TempServer
   end
 
   def destroy!
+    return if @destroy_pid != $$
     @sock.close rescue nil
     Process.kill('KILL', @pid) rescue nil
   end

data/test/test_admin.rb

@@ -148,5 +148,29 @@ class TestMogileFS__Admin < TestMogileFS
     assert_equal expected, @client.list_fids(0, 100)
   end
 
+  def test_get_devices
+    @backend.get_devices = {
+      'dev1_utilization' => '5.5',
+      'dev1_devid' => '1',
+      'dev1_hostid' => '3',
+      'dev1_observed_state' => 'writable',
+      'dev1_reject_bad_md5' => '1',
+      'dev2_utilization' => nil,
+      'dev2_devid' => '2',
+      'dev2_hostid' => '4',
+      'dev2_observed_state' => nil,
+      'devices' => '2'
+    }
+    exp = [
+      {
+        'observed_state' => 'writable',
+        'devid' => 1,
+        'reject_bad_md5' => true,
+        'utilization' => 5.5,
+        'hostid' => 3,
+      },
+      { 'devid' => 2, 'hostid' => 4 }
+    ]
+    assert_equal exp, @client.get_devices
+  end
 end
-