s3_meta_sync 0.13.1 → 0.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 216269b8e8057ed05a79cfdc52324704203c57e5
-  data.tar.gz: 924fd0b552fbe6734acb3c89e5cb3193bff08f1d
+SHA256:
+  metadata.gz: eb3dcbb8679154cb0863f3c13ba43d5c828220cbc2f0624346c19cf3617dd358
+  data.tar.gz: 6973f43e1a0bad829be2748ec65e447a2f7e96c16b2d95746d3bfad1e8daa0ea
 SHA512:
-  metadata.gz: 8ea2a75355336b6c88d303ed183d5681819c4d560f3edcde9bdfc1aca872ec690b9032abcd97e8d7c8aba36bd76bf7d2e0924903450512eb61b7642582b23134
-  data.tar.gz: 0c59e4cfb6a6630f932385e573209f644b39498336ab9fe379a104a579834518eaada8cba94dcccb4e77f910ebc719b3d1bc1779e18b1bd964ef6f64ec83d60f
+  metadata.gz: e6aca9b4086bf08b5bb18ea74622fd4ad45fa783440246499459e5d8663b331ab3b63cef7ce2c78a8eb66eb7ae6470e5fffd2beb43ab95363d38f96bf60128b5
+  data.tar.gz: d8c6e284debf19ac52094a3ec1e36cb158b318b3c655fbc5a63a9c65ebc236048c24af184327a8085acfe11b082395a53a3e199e654e63cb0ca6195b455e4a8c

data/lib/s3_meta_sync.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "optparse"
 require "s3_meta_sync/version"
 require "s3_meta_sync/syncer"

data/lib/s3_meta_sync/syncer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "net/http"
 require "open-uri"
 require "yaml"
@@ -5,17 +7,24 @@ require "digest/md5"
 require "fileutils"
 require "tmpdir"
 require "openssl"
+require "mime/types"
 
-require "aws-sdk-core"
+require "aws-sdk-s3"
 require "s3_meta_sync/zip"
 
 module S3MetaSync
   class Syncer
-    DEFAULT_REGION = 'us-east-1'
+    DEFAULT_REGION = "us-east-1"
     STAGING_AREA_PREFIX = "s3ms_"
 
+    AWS_PUBLIC_ACCESS = "public-read"
+    AWS_PRIVATE_ACCESS = "private"
+
     def initialize(config)
-      @config = config
+      @config = {
+        acl: AWS_PUBLIC_ACCESS,
+        region: DEFAULT_REGION
+      }.merge(config)
     end
 
     def sync(source, destination)
@@ -93,7 +102,7 @@ module S3MetaSync
     # Sometimes SIGTERM causes Dir.mktmpdir to not properly delete the temp folder
     # Remove 1 day old folders
     def delete_old_temp_folders
-      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + '*')
+      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + "*")
 
       day = 24 * 60 * 60
       dirs = Dir.glob(path)
@@ -161,10 +170,12 @@ module S3MetaSync
       content = Zip.zip(content) if @config[:zip] && path != META_FILE
 
       object = {
+        acl: @config[:acl],
         bucket: @bucket,
         body: content,
-        key: "#{destination}/#{path}",
-        acl: 'public-read'
+        content_encoding: content.encoding.to_s,
+        content_type: MIME::Types.of(path).first.to_s,
+        key: "#{destination}/#{path}"
       }
 
       object[:server_side_encryption] = @config[:server_side_encryption] if @config[:server_side_encryption]
@@ -195,11 +206,19 @@ module S3MetaSync
     end
 
     def s3
-      @s3 ||= ::Aws::S3::Client.new(
-        access_key_id: @config[:key],
-        secret_access_key: @config[:secret],
-        region: @config[:region] || 'us-west-2'
-      )
+      @s3 ||= begin
+        config = { region: @config[:region] }
+
+        if @config[:credentials_path]
+          config[:credentials] = Aws::SharedCredentials.new(path: @config[:credentials_path], profile_name: "default")
+        else
+          config[:access_key_id] = @config[:key]
+          config[:secret_access_key] = @config[:secret]
+          config[:session_token] = @config[:session_token] if @config[:session_token]
+        end
+
+        Aws::S3::Client.new(config)
+      end
     end
 
     def generate_meta(source)
@@ -230,21 +249,50 @@ module S3MetaSync
 
     def read_meta(source)
       file = "#{source}/#{META_FILE}"
-      parse_yaml_content(File.read(file)) if File.exist?(file)
+      if File.exist?(file)
+        content = File.read(file)
+        parse_yaml_content(content) if content.size > 0
+      end
     end
 
     def download_meta(destination)
+      if private?
+        private_access_download_meta(destination)
+      else
+        public_access_download_meta(destination)
+      end
+    end
+
+    def private_access_download_meta(destination)
+      content = private_content_download(destination, META_FILE).string
+
+      raise S3MetaSync::RemoteWithoutMeta if content.empty? # if missing, upload everything
+
+      parse_yaml_content(content)
+    rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::AccessDenied # if requesting a file that doesn't exist AccessDenied is raised
+      retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
+      retries += 1
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
+    end
+
+    def public_access_download_meta(destination)
       content = download_content("#{destination}/#{META_FILE}") { |io| io.read }
+
+      raise OpenURI::HTTPError.new("Content is empty", nil) if content.size == 0
+
       parse_yaml_content(content)
     rescue OpenURI::HTTPError
       retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
       retries += 1
-      if retries <= 1
-        sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
-        retry
-      else
-        raise RemoteWithoutMeta
-      end
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
     end
 
     def parse_yaml_content(content)
@@ -253,16 +301,30 @@ module S3MetaSync
     end
 
     def download_file(source, path, destination, zip)
-      download = download_content("#{source}/#{path}") # warning: using block form consumes more ram
-      download = Zip.unzip(download) if zip
-      path = "#{destination}/#{path}"
-      FileUtils.mkdir_p(File.dirname(path))
+      download = if private?
+        private_content_download(source, path)
+      else
+        public_content_download(source, path)
+      end
+
+      download = S3MetaSync::Zip.unzip(download) if zip
+      FileUtils.mkdir_p(File.dirname("#{destination}/#{path}"))
 
       # consumes less ram then File.write(path, content), possibly also faster
-      File.open(path, 'wb') { |f| IO.copy_stream(download, f) }
+      File.open("#{destination}/#{path}", "wb") { |f| IO.copy_stream(download, f) }
       download.close
     end
 
+    def private_content_download(source, path)
+      log "Downloading #{path}"
+      obj = s3.get_object(bucket: @bucket, key: "#{source}/#{path}")
+      obj.body
+    end
+
+    def public_content_download(source, path)
+      download_content("#{source}/#{path}") # warning: using block form consumes more ram
+    end
+
     def download_content(path)
       log "Downloading #{path}"
       url =
@@ -287,8 +349,7 @@ module S3MetaSync
         log "#{e.class} error downloading #{url}, retrying #{http_error_retries}/#{max_retries}"
         retry
       else
-        $!.message << " -- while trying to download #{url}"
-        raise
+        raise $!, "#{$!.message} -- while trying to download #{url}", $!.backtrace
       end
     rescue OpenSSL::SSL::SSLError
       ssl_error_retries ||= 0
@@ -340,5 +401,9 @@ module S3MetaSync
     def log(text, important=false)
       $stderr.puts text if @config[:verbose] or important
     end
+
+    def private?
+      @config[:acl] == AWS_PRIVATE_ACCESS
+    end
   end
 end

data/lib/s3_meta_sync/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module S3MetaSync
-  VERSION = "0.13.1"
+  VERSION = "0.15.2"
 end

data/lib/s3_meta_sync/zip.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "zlib"
 require "stringio"
 
@@ -5,7 +7,7 @@ module S3MetaSync
   module Zip
     class << self
       def zip(string)
-        io = StringIO.new("w")
+        io = StringIO.new("w".dup)
         w_gz = Zlib::GzipWriter.new(io)
         w_gz.write(string)
         w_gz.close

metadata

@@ -1,29 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.15.2
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-05 00:00:00.000000000 Z
+date: 2020-09-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mime-types
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email: michael@grosser.it
 executables:
@@ -55,8 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Sync folders with s3 using a metadata file and md5 diffs