s3_meta_sync 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73d24caf55aefff70f928dca2019bbfbfe47bc97
-  data.tar.gz: 2d57f5a45398c0cb7032d2a0747b1ffd38a7c18e
+  metadata.gz: c7a6e80c1fb50514a2c9b8d3ae92d5b64e36ea04
+  data.tar.gz: c981a2e2d8f5e66f71ee137fc5fcdd0447536ef0
 SHA512:
-  metadata.gz: f6bcbe512d940706406c791c0dde947b172672cee1d6a4f361e08bc366ad9255e640b5f1bfc97aac4bb6af33f9fee56956a9125cb9c35751f81f82b74d5b69e5
-  data.tar.gz: 359edb56cd2f2bef162f87089ca3fdcf979737ba495bd93ebc79d9aa785d2c23065307eb1add3b879d0cb64aff3851781336fef6845219372e6b999b21646be3
+  metadata.gz: f509c31ef5ed0f1ddb956b7b3d7ae8c4e714df69e69502b6f422a5095d4c494b25c128aba169c63dabab33af944504c4f84573bd79012cf0adc7ec7cfe2772f3
+  data.tar.gz: 2e60e1c6f3b49400c21b4cc25f210f2eec9a7aa1a87f00effdeb0d290bbf1674697b2b2cb46f6a6b2d8a236893800c117221cc966b01c2e928455407154002fa

data/lib/s3_meta_sync/syncer.rb

@@ -201,7 +201,7 @@ module S3MetaSync
     end
 
     def download_meta(destination)
-      content = download_content("#{destination}/#{META_FILE}")
+      content = download_content("#{destination}/#{META_FILE}") { |io| io.read }
       parse_yaml_content(content)
     rescue OpenURI::HTTPError
       retries ||= 1
@@ -220,18 +220,21 @@ module S3MetaSync
     end
 
     def download_file(source, path, destination, zip)
-      content = download_content("#{source}/#{path}")
-      content = Zip.unzip(content) if zip
-      file = "#{destination}/#{path}"
-      FileUtils.mkdir_p(File.dirname(file))
-      File.write(file, content, :encoding => content.encoding)
+      download = download_content("#{source}/#{path}") # warning: using block form consumes more ram
+      download = Zip.unzip(download) if zip
+      path = "#{destination}/#{path}"
+      FileUtils.mkdir_p(File.dirname(path))
+
+      # consumes less ram then File.write(path, content), possibly also faster
+      File.open(path, 'wb') { |f| IO.copy_stream(download, f) }
+      download.close
     end
 
     def download_content(path)
       log "Downloading #{path}"
       url = "https://s3#{"-#{region}" if region}.amazonaws.com/#{@bucket}/#{path}"
       options = (@config[:ssl_none] ? {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE} : {})
-      open(url, options).read
+      open(url, options)
     rescue OpenURI::HTTPError
       $!.message << " -- while trying to download #{url}"
       raise

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,3 @@
 module S3MetaSync
-  VERSION = "0.7.0"
+  VERSION = "0.8.1"
 end

data/lib/s3_meta_sync/zip.rb

@@ -12,8 +12,8 @@ module S3MetaSync
         io.string
       end
 
-      def unzip(string)
-        Zlib::GzipReader.new(StringIO.new(string, "rb")).read
+      def unzip(io)
+        Zlib::GzipReader.new(io)
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-12 00:00:00.000000000 Z
+date: 2016-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core