s3_meta_sync 0.14.1 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb1a0fcc2e208433541ab3517ff0659f061c8c4c073cd95402563ab1ff2bab28
-  data.tar.gz: f6fb303ad30426c77207b23ad95ac754fc03ffaa85c6eb43ddf7ced724855c14
+  metadata.gz: 54ffd2915e5dfbe58348d826c002b29391c71c8719ed6f977534f36ac86364b2
+  data.tar.gz: b512d0459d0ad43a424dcbf5cb4d8bab09a8199a285b1d365e50f38680ab9319
 SHA512:
-  metadata.gz: bd8860f96506b0f870b035e7a8c5be8a8075af282ae653928b657aad925cffbb6df17ec0829fe0e18251994905a29586ab044e5aca395ce473800238cef8a045
-  data.tar.gz: 904fc54a9e49082c1588f84d0946c8e891b854adaa33e2a09ce88b9db52ac273b3983f7be437badc44db66f134890405769d9bc8480ce14e6016bafc298d8f27
+  metadata.gz: 5a5e0d249232112ba415541f8d6185739aaf6ca913fd92d3f8e5ec1fc5af5c8eedef24398446f4b9143ab0a003eaba847e3bb1150eaef45ef146523dab82bb6a
+  data.tar.gz: 446490567aabed72d68b5ffac4d6692c31ebf1a021cc58aecfb4608013f2a00f27c5abc246626cdc433d4e1f9b7923d466b70f843c0c9da280151adf6a0a64d8

data/lib/s3_meta_sync/syncer.rb

@@ -7,17 +7,24 @@ require "digest/md5"
 require "fileutils"
 require "tmpdir"
 require "openssl"
+require "mime/types"
 
 require "aws-sdk-s3"
 require "s3_meta_sync/zip"
 
 module S3MetaSync
   class Syncer
-    DEFAULT_REGION = 'us-east-1'
+    DEFAULT_REGION = "us-east-1"
     STAGING_AREA_PREFIX = "s3ms_"
 
+    AWS_PUBLIC_ACCESS = "public-read"
+    AWS_PRIVATE_ACCESS = "private"
+
     def initialize(config)
-      @config = config
+      @config = {
+        acl: AWS_PUBLIC_ACCESS,
+        region: DEFAULT_REGION
+      }.merge(config)
     end
 
     def sync(source, destination)
@@ -95,7 +102,7 @@ module S3MetaSync
     # Sometimes SIGTERM causes Dir.mktmpdir to not properly delete the temp folder
     # Remove 1 day old folders
     def delete_old_temp_folders
-      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + '*')
+      path = File.join(Dir.tmpdir, STAGING_AREA_PREFIX + "*")
 
       day = 24 * 60 * 60
       dirs = Dir.glob(path)
@@ -163,10 +170,12 @@ module S3MetaSync
       content = Zip.zip(content) if @config[:zip] && path != META_FILE
 
       object = {
+        acl: @config[:acl],
         bucket: @bucket,
         body: content,
-        key: "#{destination}/#{path}",
-        acl: 'public-read'
+        content_encoding: content.encoding.to_s,
+        content_type: MIME::Types.of(path).first.to_s,
+        key: "#{destination}/#{path}"
       }
 
       object[:server_side_encryption] = @config[:server_side_encryption] if @config[:server_side_encryption]
@@ -197,11 +206,19 @@ module S3MetaSync
     end
 
     def s3
-      @s3 ||= Aws::S3::Client.new(
-        access_key_id: @config[:key],
-        secret_access_key: @config[:secret],
-        region: @config[:region] || 'us-west-2'
-      )
+      @s3 ||= begin
+        config = { region: @config[:region] }
+
+        if @config[:credentials_path]
+          config[:credentials] = Aws::SharedCredentials.new(path: @config[:credentials_path], profile_name: "default")
+        else
+          config[:access_key_id] = @config[:key]
+          config[:secret_access_key] = @config[:secret]
+          config[:session_token] = @config[:session_token] if @config[:session_token]
+        end
+
+        Aws::S3::Client.new(config)
+      end
     end
 
     def generate_meta(source)
@@ -239,19 +256,43 @@ module S3MetaSync
     end
 
     def download_meta(destination)
+      if private?
+        private_access_download_meta(destination)
+      else
+        public_access_download_meta(destination)
+      end
+    end
+
+    def private_access_download_meta(destination)
+      content = private_content_download(destination, META_FILE).string
+
+      raise S3MetaSync::RemoteWithoutMeta if content.empty? # if missing, upload everything
+
+      parse_yaml_content(content)
+    rescue Aws::S3::Errors::NoSuchKey, Aws::S3::Errors::AccessDenied # if requesting a file that doesn't exist AccessDenied is raised
+      retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
+      retries += 1
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
+    end
+
+    def public_access_download_meta(destination)
       content = download_content("#{destination}/#{META_FILE}") { |io| io.read }
-      raise OpenURI::HTTPError.new('Content is empty', nil) unless content.size > 0
+
+      raise OpenURI::HTTPError.new("Content is empty", nil) if content.size == 0
 
       parse_yaml_content(content)
     rescue OpenURI::HTTPError
       retries ||= 0
+
+      raise S3MetaSync::RemoteWithoutMeta if retries >= 1
+
       retries += 1
-      if retries <= 1
-        sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
-        retry
-      else
-        raise RemoteWithoutMeta
-      end
+      sleep 1 # maybe the remote meta was just updated ... give aws a second chance ...
+      retry
     end
 
     def parse_yaml_content(content)
@@ -260,16 +301,29 @@ module S3MetaSync
     end
 
     def download_file(source, path, destination, zip)
-      download = download_content("#{source}/#{path}") # warning: using block form consumes more ram
-      download = Zip.unzip(download) if zip
-      path = "#{destination}/#{path}"
-      FileUtils.mkdir_p(File.dirname(path))
+      download = if private?
+        private_content_download(source, path)
+      else
+        public_content_download(source, path)
+      end
+
+      download = S3MetaSync::Zip.unzip(download) if zip
+      FileUtils.mkdir_p(File.dirname("#{destination}/#{path}"))
 
       # consumes less ram then File.write(path, content), possibly also faster
-      File.open(path, 'wb') { |f| IO.copy_stream(download, f) }
+      File.open("#{destination}/#{path}", "wb") { |f| IO.copy_stream(download, f) }
       download.close
     end
 
+    def private_content_download(source, path)
+      obj = s3.get_object(bucket: @bucket, key: "#{source}/#{path}")
+      obj.body
+    end
+
+    def public_content_download(source, path)
+      download_content("#{source}/#{path}") # warning: using block form consumes more ram
+    end
+
     def download_content(path)
       log "Downloading #{path}"
       url =
@@ -346,5 +400,9 @@ module S3MetaSync
     def log(text, important=false)
       $stderr.puts text if @config[:verbose] or important
     end
+
+    def private?
+      @config[:acl] == AWS_PRIVATE_ACCESS
+    end
   end
 end

data/lib/s3_meta_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3MetaSync
-  VERSION = "0.14.1"
+  VERSION = "0.15.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_meta_sync
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2020-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -55,8 +55,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Sync folders with s3 using a metadata file and md5 diffs