mogilefs-client 3.7.1 → 3.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 65d3ada1142a27870a3926eeae7d15a13d28b016
+  data.tar.gz: 7ef65f2858cb0a130cbb7c0200946a7df05bf701
+SHA512:
+  metadata.gz: edecf8b75e5f3e218e63383c79da8ceaab6daed1890c3d51c0ca418936c30e6a613eb187ae5258137570eee8d649a612fcbaaf0cc606baa4165a817f146a681f
+  data.tar.gz: b9448f892913e8104c611badaf368e22acaa6faf60c70abf729200a6c0725efe2657a13498aeeb64b11b4701a62caa7f1f547eefff4c0652834745380dd04e8e

data/.document

@@ -1,6 +1,5 @@
 NEWS
 History
-ChangeLog
 LICENSE
 README
 HACKING

data/.gitignore

@@ -2,8 +2,8 @@
 /doc
 /.config
 /InstalledFiles
-/ChangeLog
-/NEWS
+/NEWS*
 /LATEST
+/GIT-VERSION-FILE
+/.manifest
 /lib/mogilefs/version.rb
-/Manifest.txt

data/{.wrongdoc.yml → .olddoc.yml}

@@ -3,3 +3,5 @@ cgit_url: http://bogomips.org/mogilefs-client.git
 git_url: git://bogomips.org/mogilefs-client.git
 rdoc_url: http://bogomips.org/mogilefs-client
 changelog_start: v1.2.1
+private_email: e@80x24.org
+public_email: mogilefs-client-public@bogomips.org

data/GIT-VERSION-GEN

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
 CONSTANT = "MogileFS::VERSION"
 RVF = "lib/mogilefs/version.rb"
-DEF_VER = "v3.7.1"
+GVF = "GIT-VERSION-FILE"
+DEF_VER = "v3.8.0"
 vn = DEF_VER
 
 # First see if there is a version file (included in release tarballs),
@@ -20,9 +21,19 @@ if File.exist?(".git")
 end
 
 vn = vn.sub!(/\Av/, "")
-new_ruby_version = "#{CONSTANT} = '#{vn}'\n"
+new_ruby_version = "#{CONSTANT} = '#{vn}' # :nodoc:\n"
 cur_ruby_version = File.read(RVF) rescue nil
 if new_ruby_version != cur_ruby_version
   File.open(RVF, "w") { |fp| fp.write(new_ruby_version) }
 end
+File.chmod(0644, RVF)
+
+# generate the makefile snippet
+new_make_version = "GIT_VERSION = #{vn}\n"
+cur_make_version = File.read(GVF) rescue nil
+if new_make_version != cur_make_version
+  File.open(GVF, "w") { |fp| fp.write(new_make_version) }
+end
+File.chmod(0644, GVF)
+
 puts vn if $0 == __FILE__

data/GNUmakefile

@@ -1,90 +1,16 @@
-# use GNU Make to run tests in parallel, and without depending on Rubygems
-all:: test
+all::
 RSYNC_DEST := bogomips.org:/srv/bogomips/mogilefs-client
-git_version_gen := $(shell ./GIT-VERSION-GEN)
+rfpackage := mogilefs-client
+pkg_extra += lib/mogilefs/version.rb
 
-T := $(wildcard test/test*.rb)
-TO := $(subst .rb,.log,$(T))
+ex := $(shell git ls-files examples/)
+doc_ex := $(addprefix doc/, $(ex))
 
-test: $(T)
-	@cat $(TO) | ruby test/aggregate.rb
-	@$(RM) $(TO)
-clean:
-	$(RM) $(TO) $(addsuffix +,$(TO))
+$(doc_ex): $(ex)
+	mkdir -p $(@D)
+	install -m 644 $< $@
+	-touch -r $< $@ # GNU
 
-
-ifndef V
-  quiet_pre = @echo '* $@';
-  quiet_post = >$(t) 2>&1
-else
-  # we can't rely on -o pipefail outside of bash 3+,
-  # so we use a stamp file to indicate success and
-  # have rm fail if the stamp didn't get created
-  stamp = $@$(log_suffix).ok
-  quiet_pre = @echo $(ruby) $@ $(TEST_OPTS); ! test -f $(stamp) && (
-  quiet_post = && > $(stamp) )>&2 | tee $(t); rm $(stamp) 2>/dev/null
-endif
-ruby = ruby
-run_test = $(quiet_pre) setsid $(ruby) -w $@ $(TEST_OPTS) $(quiet_post) || \
-  (sed "s,^,$(extra): ," >&2 < $(t); exit 1)
-
-$(T): t = $(subst .rb,.log,$@)
-$(T): export RUBYLIB := $(CURDIR)/lib:$(RUBYLIB)
-$(T):
-	$(run_test)
-
-RUBY_VERSION_FILE = lib/mogilefs/version.rb
-package:
-	git diff --exit-code HEAD^0
-	$(RM) -r pkg/
-	rake fix_perms
-	rake package
-
-libs := $(wildcard lib/*.rb lib/*/*.rb)
-flay_flags =
-flog_flags =
-flay: $(libs)
-	flay $(flay_flags) $^
-flog: $(libs)
-	flog $(flog_flags) $^
-.PHONY: $(T)
-
-check-warnings:
-	@(for i in $$(git ls-files '*.rb'| grep -v '^setup\.rb$$'); \
-	  do ruby -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
-RSYNC = rsync
-WRONGDOC = wrongdoc
-
-doc:: .document .wrongdoc.yml $(pkg_extra)
-	-find lib -type f -name '*.rbc' -exec rm -f '{}' ';'
-	$(RM) -r doc
-	$(WRONGDOC) all
-	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
-	cd doc && \
-		ln -s README README.txt && \
-		ln -s README.html README_txt.html && \
-		ln -s LICENSE LICENSE.txt && \
-		ln -s LICENSE.html LICENSE_txt.html && \
-		ln -s History History.txt && \
-		ln -s History.html History_txt.html
-
-# Create gzip variants of the same timestamp as the original so nginx
-# "gzip_static on" can serve the gzipped versions directly.
-doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
-doc_gz:
-	for i in $(docs); do \
-	  gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
-
-# this requires GNU coreutils variants
-ifneq ($(RSYNC_DEST),)
-publish_doc:
-	-git set-file-times
-	$(MAKE) doc
-	find doc/images -type f | \
-		TZ=UTC xargs touch -d '1970-01-01 00:00:06' doc/rdoc.css
-	$(MAKE) doc_gz
-	$(RSYNC) -av doc/ $(RSYNC_DEST)/
-	git ls-files | xargs touch
-endif
-
-.PHONY: doc .FORCE-GIT-VERSION-FILE
+include pkg.mk
+doc::
+	$(MAKE) $(doc_ex)

data/README

@@ -9,8 +9,10 @@ MogileFS instance.
 
 rdoc :: http://bogomips.org/mogilefs-client
 mogilefs :: http://mogilefs.org/
-list :: mailto:mogile@googlegroups.com
-email :: mailto:normalperson@yhbt.net
+list :: mailto:mogilefs-client-public@bogomips.org
+list-cc :: mailto:mogile@googlegroups.com
+list-archive :: http://bogomips.org/mogilefs-client-public
+email :: mailto:e@80x24.org
 repo :: git://bogomips.org/mogilefs-client.git
 cgit :: http://bogomips.org/mogilefs-client.git
 gitweb :: http://repo.or.cz/w/ruby-mogilefs-client.git
@@ -35,15 +37,23 @@ See MogileFS::MogileFS
 == Contact
 
 Feedback (bug reports, user/development discussion, patches, pull
-requests) are greatly appreciated and handled via email.  We currently
-piggy-back onto the public MogileFS
-mailing list at mailto:mogile@googlegroups.com for feedback.
+requests) are greatly appreciated and handled via email to a public
+inbox.  HTML email is considered spam and discarded.  No subscription
+is required or available to post:
 
-If you do not want to deal with the corporate host of the MogileFS
-mailing list, or if you wish to keep your issue secret, feel free to
-email Eric Wong at mailto:normalperson@yhbt.net.
+	mogilefs-client-public@bogomips.org
 
-Do not expect Eric Wong to read HTML mail under any circumstances.
+List archives: http://bogomips.org/mogilefs-client-public/
+
+We may also piggy-back onto the public MogileFS mailing list at
+mogile@googlegroups.com for feedback (subscription required,
+unfortunately)
+
+If you wish to keep your issue secret, feel free to email the author at:
+
+		e@80x24.org
+
+Do not expect us to read HTML mail under any circumstances.
 
 == WARNING!
 

data/TODO

@@ -3,8 +3,4 @@ private email (mailto:normalperson@yhbt.net) greatly appreciated!
 
 * MogileFS::Admin needs to be fleshed out
 
-* optional Cool.io support
-
-* optional EventMachine support
-
 * manpage for "mog"

data/examples/mog-sync.rb

@@ -0,0 +1,245 @@
+#!/usr/bin/env ruby
+usage = <<EOF
+Usage: #$0 SRC_TRACKER_LIST/SRC_DOMAIN DST_TRACKER_LIST/DST_DOMAIN"
+EOF
+Thread.abort_on_exception = $stdout.sync = $stderr.sync = true
+require 'uri'
+require 'optparse'
+require 'mogilefs'
+require 'thread'
+@verbose = 0
+copy_jobs = 1
+jobs = 1
+keep = nil
+@dryrun = false
+opts = {}
+prefix = ""
+src_class = dst_class = nil
+src_maxlen = nil
+exit_ok = true
+after = nil
+clobber_nsum = false
+
+ARGV.options do |x|
+  x.banner = usage.strip
+  x.separator ''
+  x.on('-j', '--metadata-jobs JOBS', Integer,
+       'Number of metadata jobs to run in parallel') { |n|
+    jobs = n
+  }
+  x.on('-J', '--copy-jobs JOBS', Integer,
+       'Number of copy jobs to run in parallel') { |n|
+    copy_jobs = n
+  }
+
+  x.on('-h', '--help', 'Show this help message.') { puts x; exit }
+  %w(get_file_data_timeout new_file_max_time fail_timeout timeout).each do |t|
+    x.on("--#{t.tr('_', '-')} SECONDS", Integer) { |n| opts[t.to_sym] = n }
+  end
+  x.on('-v', '--verbose') { @verbose += 1 }
+  x.on('-d', '--delete') do
+    begin
+      require 'gdbm'
+      require 'tempfile'
+      tmp = Tempfile.new(%w(mog-sync-keep .gdbm))
+      at_exit { tmp.close! }
+      keep = GDBM.new(tmp.path)
+    rescue LoadError
+      warn "gdbm extension recommended for --delete: #{e.message} (#{e.class})"
+      keep = {}
+    end
+  end
+  x.on('-n', '--dry-run') { @dryrun = opts[:readonly] = true }
+  x.on('-p', '--prefix STRING') { |s| prefix = s }
+  x.on('--src-class STRING') { |s| src_class = s }
+  x.on('--dst-class STRING') { |s| dst_class = s }
+  x.on('--after STRING') { |s| after = s }
+  x.on('--max-size STRING') { |s|
+    mult = 1
+    if s.sub!(/-1\z/, "")
+      off = -1
+    elsif s.sub!(/\+1\z/, "")
+      off = 1
+    else
+      off = 0
+    end
+    {
+      /(?:K|KiB)\z/i => 1024,
+      /(?:M|MiB)\z/i => 1024 ** 2,
+      /(?:G|GiB)\z/i => 1024 ** 3,
+      /KB\z/i => 1000,
+      /MB\z/i => 1000 ** 2,
+      /GB/i => 1000 ** 3,
+    }.each do |re, m|
+      if s.sub!(re, "")
+        mult = m
+        break
+      end
+    end
+    src_maxlen = (s.to_i * mult) + off
+  }
+  x.on('-F', '--clobber-missing-checksum') { clobber_nsum = true }
+  x.parse!
+end
+
+@verbose = 1 if @verbose == 0 && @dryrun
+ARGV.size == 2 or abort "Usage: #{usage}"
+src_spec, dst_spec = ARGV
+src_opts = opts.merge(readonly: true)
+
+def client_for(str, opts = {})
+  trackers, domain = str.split('/', 2)
+  opts[:hosts] = trackers.split(/,/)
+  opts[:domain] = domain
+  MogileFS::MogileFS.new(opts)
+end
+
+# atomic for pipes/O_APPEND:
+def warn(m); $stderr.syswrite("#{m}\n"); end
+def echo(m); $stdout.syswrite("#{m}\n"); end
+
+def copy(job_id, reason, src, dst, src_info, dst_info, dst_class)
+  key = src_info["key"]
+  length = src_info["length"]
+  unless @dryrun
+    opts = {
+      largefile: true,
+      class: dst_class || src_info["class"],
+      content_length: length,
+    }
+
+    # FIXME: test/support non-MD5 checksums
+    if /\AMD5:([a-fA-F0-9]{32})\z/ =~ src_info["checksum"]
+      md5 = [ $1 ].pack("H*")
+      opts[:content_md5] = [ md5 ].pack('m0').chomp
+    end
+    if @verbose > 1
+      echo "new_file(#{key}, #{opts.inspect})"
+    end
+    dst.new_file(key, opts) do |dst_io|
+      src.get_file_data(key, dst_io)
+    end
+  end
+  if @verbose > 0
+    echo("#{reason} #{key}")
+    if @verbose > 1 && dst_info
+      echo "I[#{job_id}] before #{dst_info.inspect}"
+      echo "I[#{job_id}]  after #{src_info.inspect}"
+    end
+  end
+  Thread.current[:mog_sync_xfer] += length
+rescue => e
+  warn "E[#{job_id}] #{e.message} (#{e.class}) (src=#{key})"
+  e.backtrace { |l| warn "E[#{job_id}] #{l}" }
+end
+
+copy_queue = SizedQueue.new(copy_jobs * 8)
+copiers = copy_jobs.times.map do |i|
+  Thread.new(i) do |job_id|
+    Thread.current[:mog_sync_xfer] = 0
+    while copy_job = copy_queue.pop
+      copy(job_id, *copy_job)
+    end
+  end
+end
+
+queue = SizedQueue.new(jobs * 8)
+consumers = jobs.times.map do |i|
+  Thread.new(i) do |job_id|
+    dst = client_for(dst_spec, opts)
+    src = client_for(src_spec, src_opts)
+    begin
+      key = queue.pop or break
+      src_info = src.file_info(key)
+      next if src_class && src_class != src_info["class"]
+      src_checksum = src_info["checksum"]
+      next if src_maxlen && src_info["length"] > src_maxlen
+
+      begin
+        # this may raise UnknownKeyError
+        dst_info = dst.file_info(key)
+
+        dst_checksum = dst_info["checksum"]
+
+        # maybe we need to wait for fsck to finish:
+        if dst_checksum == "MISSING"
+          warn "destination checksum broken #{dst_info.inspect} (skipped)"
+          next unless clobber_nsum
+        end
+
+        # tell user to fix source
+        if src_checksum == "MISSING"
+          warn "source checksum broken #{src_info.inspect} (skipped)"
+          exit_ok = false
+          next
+        end
+
+        next if dst_checksum == src_checksum
+        reason = "M"
+      rescue MogileFS::Backend::UnknownKeyError # new file
+        dst_info = nil
+        # tell user to fix source
+        if src_checksum == "MISSING"
+          warn "source checksum broken #{src_info.inspect} (copying)"
+          exit_ok = false
+        end
+        reason = "A"
+      end
+      copy_queue << [ reason, src, dst, src_info, dst_info, dst_class ]
+    rescue => e
+      warn "E[#{job_id}] #{e.message} (#{e.class}) (src=#{key})"
+      e.backtrace { |l| warn "E[#{job_id}] #{l}" }
+    end while true
+  end
+end
+
+# producer feeds consumers
+begin
+  main_src = client_for(src_spec, src_opts)
+  main_src.each_key(prefix, after: after) do |key|
+    keep[key] = "1" if keep
+    queue << key
+  end
+rescue => e
+  exit_ok = false
+  warn "Aborting due to source error: #{e.message} (#{e.class})"
+  e.backtrace { |l| warn "#{l}" }
+end
+
+# terminate producer threads
+Thread.new { consumers.each { queue << nil } }
+consumers.each { |t| t.join }
+Thread.new { copiers.each { copy_queue << nil } }
+copiers.each { |t| t.join }
+bytes_sent = copiers.inject(0) { |sent,t| sent += t[:mog_sync_xfer] }
+bytes_deleted = 0
+
+# delete is single-threaded, it is not I/O-bound and
+# we can pipeline in the future
+if keep && exit_ok
+  queue = SizedQueue.new(8)
+  deleter = Thread.new do
+    dst = client_for(dst_spec, opts)
+    while key = queue.pop
+      begin
+        dst.delete(key) unless @dryrun
+        echo "D #{key}"
+      rescue MogileFS::Backend::UnknownKeyError
+        warn "#{key} disappeared before we could delete it"
+      end
+    end
+  end
+  main_dst = client_for(dst_spec, opts)
+  main_dst.each_file_info(prefix, after: after) do |info|
+    key = info["key"]
+    next if keep.include?(key)
+    queue << key
+    bytes_deleted += info["length"]
+  end
+  queue << nil # terminate
+  deleter.join
+end
+if @verbose
+  echo "wrote #{bytes_sent} bytes, removed #{bytes_deleted} bytes"
+end
+exit(exit_ok)