s3_meta_sync 0.3.6 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4c9d56dad09b4d11098f9e57172fe7776c5d17c8
-  data.tar.gz: 9e93c8e403705699f0aa3d0d78d96d1817d53861
+  metadata.gz: 417959a794098b61bbd76e514822749834515e30
+  data.tar.gz: b4d407672cd4084340394cfcdd79bb5bba6caead
 SHA512:
-  metadata.gz: 4c79cc1844b2a247ea0fc1afc2d0e6db75836448bb048d7fd258882dcf8200ac6661afac214efc9e03b7c97eaa9ef5379c6c2a96efd33ae186fadc2ee6eb4925
-  data.tar.gz: 5120a5e78050ee13b38b12ef79b05f1ea29f5fb4e120675eeadfa6378719bb18ff493fb8ab0ee1c0e73588d1e91a4d4e48fd8ca8cc3ef2300a04433d3d0aa714
+  metadata.gz: ada88904ff3f3035975bfbb123bb5ab7d90a71a1c704c203995290ef7507ccae63875b8836148330c774d33d8d2a501160a26d36d0fd8c21a9eade8c6ad702aa
+  data.tar.gz: 4f93506f40bc7ab0f7b8381ccc42a7b334174464fc6e6dfc7f4df10af8d8f32db48f0665345e80f0486e66bee427a8c7a2387d096a385835736c55b7ef703406

data/lib/s3_meta_sync/syncer.rb

@@ -0,0 +1,251 @@
+require "open-uri"
+require "yaml"
+require "digest/md5"
+require "fileutils"
+require "tmpdir"
+
+require "aws/s3"
+
+if RUBY_VERSION < "2.0.0"
+  # need to require these or upload in multiple threads will fail on systems with high load
+  require "aws/s3/s3_object"
+  require "aws/core/response"
+  require "aws/s3/object_collection"
+end
+
+require "s3_meta_sync/zip"
+
+module S3MetaSync
+  class Syncer
+    def initialize(config)
+      @config = config
+    end
+
+    def sync(source, destination)
+      raise if source.end_with?("/") or destination.end_with?("/")
+
+      if destination.include?(":")
+        @bucket, destination = destination.split(":")
+        upload(source, destination)
+      else
+        @bucket, source = source.split(":")
+        download(source, destination)
+      end
+    end
+
+    private
+
+    def upload(source, destination)
+      corrupted = consume_corrupted_files(source)
+      remote_meta = begin
+        download_meta(destination)
+      rescue RemoteWithoutMeta
+        log "Remote has no .s3-meta-sync, uploading everything", true
+        {files: {}}
+      end
+      generate_meta(source)
+      local_files = read_meta(source)[:files]
+      remote_files = remote_meta[:files]
+      upload = if @config[:zip] == remote_meta[:zip]
+        local_files.select { |path, md5| remote_files[path] != md5 || corrupted.include?(path) }
+      else
+        local_files
+      end.map(&:first)
+      delete = remote_files.keys - local_files.keys
+      log "Uploading: #{upload.size} Deleting: #{delete.size}", true
+
+      upload_files(source, destination, upload)
+      delete_remote_files(destination, delete)
+      upload_file(source, META_FILE, destination)
+    end
+
+    def download(source, destination)
+      raise if @config[:zip]
+      remote_meta = download_meta(source)
+      generate_meta(destination)
+      local_files = read_meta(destination)[:files]
+      download = remote_meta[:files].select { |path, md5| local_files[path] != md5 }.map(&:first)
+      delete = local_files.keys - remote_meta[:files].keys
+
+      log "Downloading: #{download.size} Deleting: #{delete.size}", true
+
+      unless download.empty? && delete.empty?
+        Dir.mktmpdir do |staging_area|
+          copy_content(destination, staging_area)
+          download_files(source, staging_area, download, remote_meta[:zip])
+          delete_local_files(staging_area, delete)
+          download_file(source, META_FILE, staging_area, false)
+          verify_integrity!(staging_area, destination)
+          delete_empty_folders(staging_area)
+          self.class.swap_in_directory(destination, staging_area)
+          FileUtils.mkdir(staging_area)
+        end
+      end
+    end
+
+    def copy_content(destination, dir)
+      system "cp -R #{destination}/* #{dir} 2>/dev/null"
+    end
+
+    # almost atomic when destination and temp dir are not on the same device
+    def self.swap_in_directory(destination, dir)
+      next_dir = "#{destination}-next"
+      delete = "#{destination}-delete"
+
+      # clean up potential leftovers from last run
+      FileUtils.remove_dir(next_dir) if File.exist?(next_dir)
+      FileUtils.remove_dir(delete) if File.exist?(delete)
+
+      # move onto the same device
+      FileUtils.mv(dir, next_dir)
+
+      # copy permissions
+      FileUtils.chmod_R(File.stat(destination).mode, next_dir)
+
+      # swap
+      FileUtils.mv(destination, delete)
+      FileUtils.mv(next_dir, destination)
+
+      # cleanup old
+      FileUtils.remove_dir(delete)
+    end
+
+    def verify_integrity!(staging_area, destination)
+      file = "#{staging_area}/#{META_FILE}"
+      remote = YAML.load_file(file)[:files]
+      actual = meta_data(staging_area)[:files]
+
+      if remote != actual
+        corrupted = actual.select { |file, md5| remote[file] && remote[file] != md5 }.map(&:first)
+        File.write("#{destination}/#{CORRUPT_FILES_LOG}", corrupted.join("\n"))
+        log "corrupted files downloaded:\n#{corrupted.join("\n")}", true
+        raise RemoteCorrupt
+      end
+    end
+
+    def consume_corrupted_files(source)
+      log = "#{source}/#{CORRUPT_FILES_LOG}"
+      if File.exist?(log)
+        corrupted = File.read(log).split("\n")
+        log "force uploading #{corrupted.size} corrupted files", true
+        File.unlink log
+        corrupted
+      else
+        []
+      end
+    end
+
+    def upload_file(source, path, destination)
+      log "Uploading #{path}"
+      content = File.read("#{source}/#{path}")
+      content = Zip.zip(content) if @config[:zip]
+      s3.objects["#{destination}/#{path}"].write content, :acl => :public_read
+    end
+
+    def delete_remote_files(remote, paths)
+      paths.each { |path| log "Deleting #{@bucket}:#{remote}/#{path}" }
+      s3.objects.delete paths.map { |path| "#{remote}/#{path}" }
+    end
+
+    def delete_local_files(local, paths)
+      paths = paths.map { |path| "#{local}/#{path}" }
+      paths.each { |path| log "Deleting #{path}" }
+      File.delete(*paths)
+    end
+
+    def s3
+      @s3 ||= ::AWS::S3.new(
+        access_key_id: @config[:key],
+        secret_access_key: @config[:secret]
+      ).buckets[@bucket]
+    end
+
+    def generate_meta(source)
+      file = "#{source}/#{META_FILE}"
+      FileUtils.mkdir_p(File.dirname(file))
+      File.write(file, meta_data(source).to_yaml)
+    end
+
+    def meta_data(source)
+      return {} unless File.directory?(source)
+      files = Dir.chdir(source) do
+        files = Dir["**/*"].select { |f| File.file?(f) }
+        Hash[files.map { |file| [file, Digest::MD5.file(file).to_s] }]
+      end
+      {files: files}
+    end
+
+    def read_meta(source)
+      file = "#{source}/#{META_FILE}"
+      File.exist?(file) ? YAML.load(File.read(file)) : {}
+    end
+
+    def download_meta(destination)
+      content = download_content("#{destination}/#{META_FILE}")
+      result = YAML.load(content)
+      result.key?(:files) ? result : {files: result} # support new an old format
+    rescue
+      raise RemoteWithoutMeta
+    end
+
+    def download_file(source, path, destination, zip)
+      content = download_content("#{source}/#{path}")
+      content = Zip.unzip(content) if zip
+      file = "#{destination}/#{path}"
+      FileUtils.mkdir_p(File.dirname(file))
+      File.write(file, content, :encoding => content.encoding)
+    end
+
+    def download_content(path)
+      log "Downloading #{path}"
+      url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
+      options = (@config[:ssl_none] ? {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE} : {})
+      open(url, options).read
+    rescue OpenURI::HTTPError
+      raise "Unable to download #{url} -- #{$!}"
+    rescue OpenSSL::SSL::SSLError
+      retries ||= 0
+      retries += 1
+      if retries == 1
+        log "SSL error downloading #{path}, retrying"
+        retry
+      else
+        raise
+      end
+    end
+
+    def delete_empty_folders(destination)
+      `find #{destination} -depth -empty -delete`
+    end
+
+    def download_files(source, destination, paths, zip)
+      in_multiple_threads(paths) do |path|
+        download_file(source, path, destination, zip)
+      end
+    end
+
+    def upload_files(source, destination, paths)
+      in_multiple_threads(paths) { |path| upload_file(source, path, destination) }
+    end
+
+    def region
+      @config[:region] unless @config[:region].to_s.empty?
+    end
+
+    def in_multiple_threads(data)
+      threads = [@config[:parallel] || 10, data.size].min
+      data = data.dup
+      (0...threads).to_a.map do
+        Thread.new do
+          while slice = data.shift
+            yield slice
+          end
+        end
+      end.each(&:join)
+    end
+
+    def log(text, important=false)
+      $stderr.puts text if @config[:verbose] or important
+    end
+  end
+end

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.3.6"
+  VERSION = "0.4.0"
 end

data/lib/s3_meta_sync/zip.rb

@@ -0,0 +1,20 @@
+require "zlib"
+require "stringio"
+
+module S3MetaSync
+  module Zip
+    class << self
+      def zip(string)
+        io = StringIO.new("w")
+        w_gz = Zlib::GzipWriter.new(io)
+        w_gz.write(string)
+        w_gz.close
+        io.string
+      end
+
+      def unzip(string)
+        Zlib::GzipReader.new(StringIO.new(string, "rb")).read
+      end
+    end
+  end
+end

data/lib/s3_meta_sync.rb

@@ -1,19 +1,6 @@
-require "s3_meta_sync/version"
-require "open-uri"
-require "yaml"
-require "digest/md5"
 require "optparse"
-require "fileutils"
-require "tmpdir"
-
-require "aws/s3"
-
-if RUBY_VERSION < "2.0.0"
-  # need to require these or upload in multiple threads will fail on systems with high load
-  require "aws/s3/s3_object"
-  require "aws/core/response"
-  require "aws/s3/object_collection"
-end
+require "s3_meta_sync/version"
+require "s3_meta_sync/syncer"
 
 module S3MetaSync
   RemoteWithoutMeta = Class.new(Exception)
@@ -21,223 +8,6 @@ module S3MetaSync
   META_FILE = ".s3-meta-sync"
   CORRUPT_FILES_LOG = "s3-meta-sync-corrupted.log"
 
-  class Syncer
-    def initialize(config)
-      @config = config
-    end
-
-    def sync(source, destination)
-      raise if source.end_with?("/") or destination.end_with?("/")
-
-      if destination.include?(":")
-        @bucket, destination = destination.split(":")
-        upload(source, destination)
-      else
-        @bucket, source = source.split(":")
-        download(source, destination)
-      end
-    end
-
-    private
-
-    def upload(source, destination)
-      corrupted = consume_corrupted_files(source)
-      remote_info = begin
-        download_meta(destination)
-      rescue RemoteWithoutMeta
-        log "Remote has no .s3-meta-sync, uploading everything", true
-        {}
-      end
-      generate_meta(source)
-      local_info = read_meta(source)
-      upload = local_info.select { |path, md5| remote_info[path] != md5 || corrupted.include?(path) }.map(&:first)
-      delete = remote_info.keys - local_info.keys
-      log "Uploading: #{upload.size} Deleting: #{delete.size}", true
-
-      upload_files(source, destination, upload)
-      delete_remote_files(destination, delete)
-      upload_file(source, META_FILE, destination)
-    end
-
-    def download(source, destination)
-      remote_info = download_meta(source)
-      generate_meta(destination)
-      local_info = read_meta(destination)
-      download = remote_info.select { |path, md5| local_info[path] != md5 }.map(&:first)
-      delete = local_info.keys - remote_info.keys
-
-      log "Downloading: #{download.size} Deleting: #{delete.size}", true
-
-      unless download.empty? && delete.empty?
-        Dir.mktmpdir do |staging_area|
-          copy_content(destination, staging_area)
-          download_files(source, staging_area, download)
-          delete_local_files(staging_area, delete)
-          download_file(source, META_FILE, staging_area)
-          verify_integrity!(staging_area, destination)
-          delete_empty_folders(staging_area)
-          self.class.swap_in_directory(destination, staging_area)
-          FileUtils.mkdir(staging_area)
-        end
-      end
-    end
-
-    def copy_content(destination, dir)
-      system "cp -R #{destination}/* #{dir} 2>/dev/null"
-    end
-
-    # almost atomic when destination and temp dir are not on the same device
-    def self.swap_in_directory(destination, dir)
-      next_dir = "#{destination}-next"
-      delete = "#{destination}-delete"
-
-      # clean up potential leftovers from last run
-      FileUtils.remove_dir(next_dir) if File.exist?(next_dir)
-      FileUtils.remove_dir(delete) if File.exist?(delete)
-
-      # move onto the same device
-      FileUtils.mv(dir, next_dir)
-
-      # copy permissions
-      FileUtils.chmod_R(File.stat(destination).mode, next_dir)
-
-      # swap
-      FileUtils.mv(destination, delete)
-      FileUtils.mv(next_dir, destination)
-
-      # cleanup old
-      FileUtils.remove_dir(delete)
-    end
-
-    def verify_integrity!(staging_area, destination)
-      file = "#{staging_area}/#{META_FILE}"
-      remote = YAML.load_file(file)
-      actual = meta_data(staging_area)
-
-      if remote != actual
-        corrupted = actual.select { |file, md5| remote[file] && remote[file] != md5 }.map(&:first)
-        File.write("#{destination}/#{CORRUPT_FILES_LOG}", corrupted.join("\n"))
-        log "corrupted files downloaded:\n#{corrupted.join("\n")}", true
-        raise RemoteCorrupt
-      end
-    end
-
-    def consume_corrupted_files(source)
-      log = "#{source}/#{CORRUPT_FILES_LOG}"
-      if File.exist?(log)
-        corrupted = File.read(log).split("\n")
-        log "force uploading #{corrupted.size} corrupted files", true
-        File.unlink log
-        corrupted
-      else
-        []
-      end
-    end
-
-    def upload_file(source, path, destination)
-      log "Uploading #{path}"
-      s3.objects["#{destination}/#{path}"].write File.read("#{source}/#{path}"), :acl => :public_read
-    end
-
-    def delete_remote_files(remote, paths)
-      paths.each { |path| log "Deleting #{@bucket}:#{remote}/#{path}" }
-      s3.objects.delete paths.map { |path| "#{remote}/#{path}" }
-    end
-
-    def delete_local_files(local, paths)
-      paths = paths.map { |path| "#{local}/#{path}" }
-      paths.each { |path| log "Deleting #{path}" }
-      File.delete(*paths)
-    end
-
-    def s3
-      @s3 ||= ::AWS::S3.new(:access_key_id => @config[:key], :secret_access_key => @config[:secret]).buckets[@bucket]
-    end
-
-    def generate_meta(source)
-      file = "#{source}/#{META_FILE}"
-      FileUtils.mkdir_p(File.dirname(file))
-      File.write(file, meta_data(source).to_yaml)
-    end
-
-    def meta_data(source)
-      return {} unless File.directory?(source)
-      Dir.chdir(source) do
-        files = Dir["**/*"].select { |f| File.file?(f) }
-        Hash[files.map { |file| [file, Digest::MD5.file(file).to_s] }]
-      end
-    end
-
-    def read_meta(source)
-      file = "#{source}/#{META_FILE}"
-      File.exist?(file) ? YAML.load(File.read(file)) : {}
-    end
-
-    def download_meta(destination)
-      content = download_content("#{destination}/#{META_FILE}")
-      YAML.load(content)
-    rescue
-      raise RemoteWithoutMeta
-    end
-
-    def download_file(source, path, destination)
-      content = download_content("#{source}/#{path}")
-      file = "#{destination}/#{path}"
-      FileUtils.mkdir_p(File.dirname(file))
-      File.write(file, content, :encoding => content.encoding)
-    end
-
-    def download_content(path)
-      log "Downloading #{path}"
-      url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
-      options = (@config[:ssl_none] ? {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE} : {})
-      open(url, options).read
-    rescue OpenURI::HTTPError
-      raise "Unable to download #{url} -- #{$!}"
-    rescue OpenSSL::SSL::SSLError
-      retries ||= 0
-      retries += 1
-      if retries == 1
-        log "SSL error downloading #{path}, retrying"
-        retry
-      else
-        raise
-      end
-    end
-
-    def delete_empty_folders(destination)
-      `find #{destination} -depth -empty -delete`
-    end
-
-    def download_files(source, destination, paths)
-      in_multiple_threads(paths) { |path| download_file(source, path, destination) }
-    end
-
-    def upload_files(source, destination, paths)
-      in_multiple_threads(paths) { |path| upload_file(source, path, destination) }
-    end
-
-    def region
-      @config[:region] unless @config[:region].to_s.empty?
-    end
-
-    def in_multiple_threads(data)
-      threads = [@config[:parallel] || 10, data.size].min
-      data = data.dup
-      (0...threads).to_a.map do
-        Thread.new do
-          while slice = data.shift
-            yield slice
-          end
-        end
-      end.each(&:join)
-    end
-
-    def log(text, important=false)
-      $stderr.puts text if @config[:verbose] or important
-    end
-  end
-
   class << self
     def run(argv)
       source, dest, options = parse_options(argv)
@@ -247,8 +17,9 @@ module S3MetaSync
 
     def parse_options(argv)
       options = {
-        :key => ENV["AWS_ACCESS_KEY_ID"],
-        :secret => ENV["AWS_SECRET_ACCESS_KEY"]
+        key: ENV["AWS_ACCESS_KEY_ID"],
+        secret: ENV["AWS_SECRET_ACCESS_KEY"],
+        zip: false,
       }
       OptionParser.new do |opts|
         opts.banner = <<-BANNER.gsub(/^ {10}/, "")
@@ -268,7 +39,8 @@ module S3MetaSync
         opts.on("-s", "--secret SECRET", "AWS secret key") { |c| options[:secret] = c }
         opts.on("-r", "--region REGION", "AWS region if not us-standard") { |c| options[:region] = c }
         opts.on("-p", "--parallel COUNT", Integer, "Use COUNT threads for download/upload default: 10") { |c| options[:parallel] = c }
-        opts.on("--ssl-none", "Do not verify ssl certs") { |c| options[:ssl_none] = true }
+        opts.on("--ssl-none", "Do not verify ssl certs") { options[:ssl_none] = true }
+        opts.on("-z", "--zip", "Zip when uploading to save bandwidth") { options[:zip] = true }
         opts.on("-V", "--verbose", "Verbose mode"){ options[:verbose] = true }
         opts.on("-h", "--help", "Show this.") { puts opts; exit }
         opts.on("-v", "--version", "Show Version"){ puts VERSION; exit}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.4.0
 platform: ruby
 authors:
 - Michael Grosser
@@ -30,7 +30,7 @@ cert_chain:
   F5etKHZg0j3eHO31/i2HnswY04lqGImUu6aM5EnijFTB7PPW2KwKKM4+kKDYFdlw
   /0WV1Ng2/Y6qsHwmqGg2VlYj2h4=
   -----END CERTIFICATE-----
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2014-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -55,7 +55,9 @@ extra_rdoc_files: []
 files:
 - bin/s3-meta-sync
 - lib/s3_meta_sync.rb
+- lib/s3_meta_sync/syncer.rb
 - lib/s3_meta_sync/version.rb
+- lib/s3_meta_sync/zip.rb
 homepage: http://github.com/grosser/s3_meta_sync
 licenses:
 - MIT