s3_meta_sync 0.13.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5a615961aa602927cb18e0e70d56be412f6dee9e
-  data.tar.gz: 745648bd849c20d6ad82f2617fe5363d1f5218a9
+SHA256:
+  metadata.gz: 0bb60ae142b42d7c3055e17872c6a512c37aade47ac5773f9e321c002c6f0a7e
+  data.tar.gz: 8cde3dd1297a970208c2acc42a21488a979ff3248806694648699606b1d76913
 SHA512:
-  metadata.gz: 523b3f6b26a1d690be90cb2af4762560a8ddee28da87bf222d82089b789f9ea824bc06ec574fd42a013f8f4732eafeb60e58d3b30061719e91d2472c3c104f25
-  data.tar.gz: 351f466424af9a242b9f29b323bbcba66d6347581a1da412bb0a53da4f33c89d1cecaecd58a08ff8678ceed8264afc9ef957874a8d26de492e022496b07494de
+  metadata.gz: be936302e585df271ada643c3f3ba13d54014957659ee3f8a06f9d3b35f4d15df542b32367dd7f31d5086e56ad84ea92753754a3db39f5042bf6f8da3fb0d750
+  data.tar.gz: 584a30fe6eaee8ce3b91f021a7a7afe4ab10620635a540598a3f8c3d833229a0cf1cfdb1099a6bd761dc09e838bff4df517073ddcadfeb17a6fd5b0e122e0c4e

data/lib/s3_meta_sync.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "s3_meta_sync/version"
 require "s3_meta_sync/syncer"

data/lib/s3_meta_sync/syncer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "net/http"
 require "open-uri"
 require "yaml"
@@ -5,17 +7,24 @@ require "digest/md5"
 require "fileutils"
 require "tmpdir"
 require "openssl"
+require "mime/types"
 
-require "aws-sdk-core"
+require "aws-sdk-s3"
 require "s3_meta_sync/zip"
 
 module S3MetaSync
   class Syncer
-    DEFAULT_REGION = 'us-east-1'
+    DEFAULT_REGION = "us-east-1"
     STAGING_AREA_PREFIX = "s3ms_"
 
+    AWS_PUBLIC_ACCESS = "public-read"
+    AWS_PRIVATE_ACCESS = "private"
+
     def initialize(config)
-      @config = config
+      @config = {
+        acl: AWS_PUBLIC_ACCESS,
+        region: DEFAULT_REGION
+      }.merge(config)
     end
 
     def sync(source, destination)
@@ -93,7 +102,7 @@ module S3MetaSync
     # Sometimes SIGTERM causes Dir.mktmpdir to not properly delete the temp folder
     # Remove 1 day old folders
     def delete_old_temp_folders
-      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + '*')
+      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + "*")
 
       day = 24 * 60 * 60
       dirs = Dir.glob(path)
@@ -161,10 +170,12 @@ module S3MetaSync
       content = Zip.zip(content) if @config[:zip] && path != META_FILE
 
       object = {
+        acl: @config[:acl],
         bucket: @bucket,
         body: content,
-        key: "#{destination}/#{path}",
-        acl: 'public-read'
+        content_encoding: content.encoding.to_s,
+        content_type: MIME::Types.of(path).first.to_s,
+        key: "#{destination}/#{path}"
       }
 
       object[:server_side_encryption] = @config[:server_side_encryption] if @config[:server_side_encryption]
@@ -175,11 +186,15 @@ module S3MetaSync
     def delete_remote_files(remote, paths)
       paths.each { |path| log "Deleting #{@bucket}:#{remote}/#{path}" }
       if paths.any?
-        s3.delete_objects(
-          delete: { objects: paths.map { |path| {key: "#{remote}/#{path}"} } },
-          request_payer: "requester",
-          bucket: @bucket
-        )
+        # keys are limited to 1000 per request: http://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Bucket.html#delete_objects-instance_method
+        paths.each_slice(1000) do |sliced_paths|
+          log "Sending request for #{sliced_paths.size} keys"
+          s3.delete_objects(
+            delete: { objects: sliced_paths.map { |path| {key: "#{remote}/#{path}"} } },
+            request_payer: "requester",
+            bucket: @bucket
+          )
+        end
       end
     end
 
@@ -191,11 +206,19 @@ module S3MetaSync
     end
 
     def s3
-      @s3 ||= ::Aws::S3::Client.new(
-        access_key_id: @config[:key],
-        secret_access_key: @config[:secret],
-        region: @config[:region] || 'us-west-2'
-      )
+      @s3 ||= begin
+        config = { region: @config[:region] }
+
+        if @config[:credentials_path]
+          config[:credentials] = Aws::SharedCredentials.new(path: @config[:credentials_path], profile_name: "default")
+        else
+          config[:access_key_id] = @config[:key]
+          config[:secret_access_key] = @config[:secret]
+          config[:session_token] = @config[:session_token] if @config[:session_token]
+        end
+
+        Aws::S3::Client.new(config)
+      end
     end
 
     def generate_meta(source)
@@ -226,21 +249,50 @@ module S3MetaSync
 
     def read_meta(source)
       file = "#{source}/#{META_FILE}"
-      parse_yaml_content(File.read(file)) if File.exist?(file)
+      if File.exist?(file)
+        content = File.read(file)
+        parse_yaml_content(content) if content.size > 0
+      end
     end
 
     def download_meta(destination)
+      if private?
+        private_access_download_meta(destination)
+      else
+        public_access_download_meta(destination)
+      end
+    end
+
+    def private_access_download_meta(destination)
+      content = private_content_download(destination, META_FILE).string
+
+      raise S3MetaSync::RemoteWithoutMeta if content.empty? # if missing, upload everything
+
+      parse_yaml_content(content)
+    rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::AccessDenied # if requesting a file that doesn't exist AccessDenied is raised
+      retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
+      retries += 1
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
+    end
+
+    def public_access_download_meta(destination)
       content = download_content("#{destination}/#{META_FILE}") { |io| io.read }
+
+      raise OpenURI::HTTPError.new("Content is empty", nil) if content.size == 0
+
       parse_yaml_content(content)
     rescue OpenURI::HTTPError
       retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
       retries += 1
-      if retries <= 1
-        sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
-        retry
-      else
-        raise RemoteWithoutMeta
-      end
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
     end
 
     def parse_yaml_content(content)
@@ -249,16 +301,29 @@ module S3MetaSync
     end
 
     def download_file(source, path, destination, zip)
-      download = download_content("#{source}/#{path}") # warning: using block form consumes more ram
-      download = Zip.unzip(download) if zip
-      path = "#{destination}/#{path}"
-      FileUtils.mkdir_p(File.dirname(path))
+      download = if private?
+        private_content_download(source, path)
+      else
+        public_content_download(source, path)
+      end
+
+      download = S3MetaSync::Zip.unzip(download) if zip
+      FileUtils.mkdir_p(File.dirname("#{destination}/#{path}"))
 
       # consumes less ram then File.write(path, content), possibly also faster
-      File.open(path, 'wb') { |f| IO.copy_stream(download, f) }
+      File.open("#{destination}/#{path}", "wb") { |f| IO.copy_stream(download, f) }
       download.close
     end
 
+    def private_content_download(source, path)
+      obj = s3.get_object(bucket: @bucket, key: "#{source}/#{path}")
+      obj.body
+    end
+
+    def public_content_download(source, path)
+      download_content("#{source}/#{path}") # warning: using block form consumes more ram
+    end
+
     def download_content(path)
       log "Downloading #{path}"
       url =
@@ -283,8 +348,7 @@ module S3MetaSync
         log "#{e.class} error downloading #{url}, retrying #{http_error_retries}/#{max_retries}"
         retry
       else
-        $!.message << " -- while trying to download #{url}"
-        raise
+        raise $!, "#{$!.message} -- while trying to download #{url}", $!.backtrace
       end
     rescue OpenSSL::SSL::SSLError
       ssl_error_retries ||= 0
@@ -336,5 +400,9 @@ module S3MetaSync
     def log(text, important=false)
       $stderr.puts text if @config[:verbose] or important
     end
+
+    def private?
+      @config[:acl] == AWS_PRIVATE_ACCESS
+    end
   end
 end

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module S3MetaSync
-  VERSION = "0.13.0"
+  VERSION = "0.15.1"
 end

data/lib/s3_meta_sync/zip.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "zlib"
 require "stringio"
 
@@ -5,7 +7,7 @@ module S3MetaSync
   module Zip
     class << self
       def zip(string)
-        io = StringIO.new("w")
+        io = StringIO.new("w".dup)
         w_gz = Zlib::GzipWriter.new(io)
         w_gz.write(string)
         w_gz.close

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.15.1
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-05 00:00:00.000000000 Z
+date: 2020-08-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: michael@grosser.it
 executables:
@@ -55,8 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Sync folders with s3 using a metadata file and md5 diffs