capistrano-s3_archive 0.9.9 → 1.0.0

data/lib/capistrano/scm/s3_archive/archive_object.rb

@@ -0,0 +1,76 @@
+require "aws-sdk-s3"
+require "uri"
+
+module Capistrano
+  class SCM
+    class S3Archive
+      class ArchiveObject
+        attr_reader :bucket, :prefix, :version_id, :sort_proc, :branch, :client
+
+        def initialize(repo_url: nil, version_id: nil, sort_proc: nil, branch: :latest, client_options: {})
+          uri = URI.parse(repo_url)
+          @bucket = uri.host
+          @prefix = uri.path.sub(%r{/?\Z}, '/').slice(1..-1) # normalize path
+          @version_id = version_id
+          @sort_proc = sort_proc
+          @branch = branch
+          @client = Aws::S3::Client.new(client_options)
+        end
+
+        def check_access!
+          client.list_objects(bucket: bucket, prefix: prefix)
+        end
+
+        def key
+          @key ||= case branch.to_sym
+                   when :master, :latest
+                     latest_key
+                   else
+                     prefix + branch.to_s
+                   end
+        end
+
+        def key_basename
+          File.basename(key)
+        end
+
+        def latest_key
+          list_all_objects.min(&sort_proc).key
+        end
+
+        def list_all_objects
+          response = client.list_objects(bucket: bucket, prefix: prefix)
+          response.inject([]) do |objects, page|
+            objects + page.contents
+          end
+        end
+
+        def etag
+          metadata.tap { |it| raise "No such object: #{current_revision}" if it.nil? }.etag
+        end
+
+        def current_revision
+          if version_id
+            "#{key}?versionid=#{version_id}"
+          else
+            key
+          end
+        end
+
+        def metadata
+          client.list_object_versions(bucket: bucket, prefix: key).versions.find do |v|
+            if version_id then v.version_id == version_id
+            else v.is_latest
+            end
+          end
+        end
+
+        def get_object(io)
+          options = { bucket: bucket, key: key }
+          options[:version_id] = version_id if version_id
+          client.get_object(options, target: io)
+        end
+      end
+    end
+  end
+end

data/lib/capistrano/scm/s3_archive/local_cache.rb

@@ -0,0 +1,82 @@
+module Capistrano
+  class SCM
+    class S3Archive
+      class LocalCache
+        attr_reader :backend, :download_dir, :cache_dir, :archive_object
+        include FileUtils
+        class ResourceBusyError < StandardError; end
+
+        def initialize(backend, download_dir, cache_dir, archive_object)
+          @backend = backend
+          @download_dir = download_dir
+          @cache_dir = cache_dir
+          @archive_object = archive_object
+        end
+
+        def download
+          download_lock do
+            tmp_file = "#{target_file}.part"
+            etag_file = File.join(download_dir, ".#{archive_object.key_basename}.etag")
+            raise "#{tmp_file} is found. Another process is running?" if File.exist?(tmp_file)
+
+            if all_file_exist?([target_file, etag_file]) && File.read(etag_file) == archive_object.etag
+              backend.info "#{target_file} (etag:#{archive_object.etag}) is found. download skipped."
+            else
+              backend.info "Download s3://#{archive_object.bucket}/#{archive_object.key} to #{target_file}"
+              mkdir_p(File.dirname(target_file))
+              File.open(tmp_file, 'w') do |file|
+                archive_object.get_object(file)
+              end
+              move(tmp_file, target_file)
+              File.write(etag_file, archive_object.etag)
+            end
+          end
+        end
+
+        def extract
+          remove_entry_secure(cache_dir) if File.exist?(cache_dir)
+          mkdir_p(cache_dir)
+          case target_file
+          when /\.zip\?.*\Z/
+            cmd = "unzip -q -d #{cache_dir} #{target_file}"
+          when /\.tar\.gz\?.*\Z|\.tar\.bz2\?.*\Z|\.tgz\?.*\Z/
+            cmd = "tar xf #{target_file} -C #{cache_dir}"
+          end
+
+          backend.execute cmd # should I use `execute`?
+        end
+
+        def cleanup(keep: 0)
+          downloaded_files = Dir.glob(download_dir).sort_by(&File.method(:mtime))
+          return if downloaded_files.count <= keep
+
+          to_be_removes = (downloaded_files - downloaded_files.last(keep)).flat_map { |f| [f, ".#{f}.etag"] }
+          remove(to_be_removes, force: true)
+        end
+
+        def target_file
+          basename = [archive_object.key_basename, archive_object.version_id].join('?')
+          File.join(download_dir, basename)
+        end
+
+        def all_file_exist?(arr)
+          arr.all?(&File.method(:exist?))
+        end
+
+        def download_lock(&block)
+          mkdir_p(File.dirname(download_dir))
+          lockfile = "#{download_dir}.lock"
+          begin
+            File.open(lockfile, "w") do |file|
+              raise ResourceBusyError, "Could not get #{lockfile}" unless file.flock(File::LOCK_EX | File::LOCK_NB)
+
+              block.call
+            end
+          ensure
+            rm lockfile if File.exist? lockfile
+          end
+        end
+      end
+    end
+  end
+end

data/lib/capistrano/scm/s3_archive/remote_cache.rb

@@ -0,0 +1,45 @@
+module Capistrano
+  class SCM
+    class S3Archive
+      class RemoteCache
+        attr_reader :backend, :download_dir, :archive_object
+
+        def initialize(backend, download_dir, archive_object)
+          @backend = backend
+          @download_dir = download_dir
+          @archive_object = archive_object
+        end
+
+        def download
+          tmp_file = "#{target_file}.part"
+          etag_file = File.join(download_dir, ".#{archive_object.key_basename}.etag")
+          if backend.test("[ -f #{target_file} -a -f #{etag_file} ]") &&
+             backend.capture(:cat, etag_file) == archive_object.etag
+            backend.info "#{target_file} (etag:#{archive_object.etag}) is found. download skipped."
+          else
+            backend.info "Download s3://#{archive_object.bucket}/#{archive_object.key} to #{target_file}"
+            backend.execute(:mkdir, "-p", download_dir)
+            backend.execute(:aws, *['s3api', 'get-object', "--bucket #{archive_object.bucket}", "--key #{archive_object.key}", archive_object.version_id ? "--version-id #{archive_object.version_id}" : nil, tmp_file].compact)
+            backend.execute(:mv, tmp_file, target_file)
+            backend.execute(:echo, "-n", "'#{archive_object.etag}'", "|tee", etag_file)
+          end
+        end
+
+        def cleanup(keep: 0)
+          downloaded_files = backend.capture(:ls, "-xtr", download_dir).split
+          return if downloaded_files.count <= keep
+
+          to_be_removes = (downloaded_files - downloaded_files.last(keep)).flat_map do |file|
+            [File.join(download_dir, file), File.join(download_dir, ".#{f}.etag")]
+          end
+          backend.execute(:rm, '-f', *to_be_removes)
+        end
+
+        def target_file
+          basename = [archive_object.key_basename, archive_object.version_id].join('?')
+          File.join(download_dir, basename)
+        end
+      end
+    end
+  end
+end

data/lib/capistrano/scm/s3_archive.rb

@@ -1,20 +1,40 @@
 require "capistrano/scm/plugin"
-require "aws-sdk"
-require "uri"
 
 module Capistrano
   class SCM
     class S3Archive < Capistrano::SCM::Plugin
-      attr_reader :extractor
-      include FileUtils
+    end
+  end
+end
 
-      class ResourceBusyError < StandardError; end
+require "capistrano/scm/s3_archive/archive_object"
+require "capistrano/scm/s3_archive/local_cache"
+require "capistrano/scm/s3_archive/remote_cache"
+
+module Capistrano
+  class SCM
+    class S3Archive
+      def define_tasks
+        eval_rakefile File.expand_path("tasks/s3_archive.rake", __dir__)
+      end
+
+      def register_hooks
+        after "deploy:new_release_path", "s3_archive:create_release"
+        before "deploy:check", "s3_archive:check"
+        before "deploy:set_current_revision", "s3_archive:set_current_revision"
+      end
 
       def set_defaults
         set_if_empty :s3_archive_client_options, {}
-        set_if_empty :s3_archive_extract_to, :local # :local or :remote
         set_if_empty(:s3_archive_sort_proc, ->(new, old) { old.key <=> new.key })
+        set_if_empty :s3_archive_strategy, :rsync
         set_if_empty :s3_archive_object_version_id, nil
+
+        # strategy direct (alpha)
+        set_if_empty :s3_archive_remote_cache_dir, -> { File.join(shared_path, "archives") }
+
+        # strategy rsync
+        set_if_empty :s3_archive_skip_download, nil
         set_if_empty :s3_archive_local_download_dir, "tmp/archives"
         set_if_empty :s3_archive_local_cache_dir, "tmp/deploy"
         set_if_empty :s3_archive_remote_rsync_options, ['-az', '--delete']
@@ -22,268 +42,115 @@ module Capistrano
         set_if_empty :s3_archive_remote_rsync_runner_options, {}
         set_if_empty :s3_archive_rsync_cache_dir, "shared/deploy"
         set_if_empty :s3_archive_hardlink_release, false
-        # internal use
-        set_if_empty :s3_archive_rsync_copy, "rsync --archive --acls --xattrs"
-      end
-
-      def define_tasks
-        eval_rakefile File.expand_path("../tasks/s3_archive.rake", __FILE__)
-      end
-
-      def register_hooks
-        after "deploy:new_release_path", "s3_archive:create_release"
-        before "deploy:check", "s3_archive:check"
-        before "deploy:set_current_revision", "s3_archive:set_current_revision"
+        set_if_empty :s3_archive_remote_rsync_copy_option, "--archive --acls --xattrs"
       end
 
+      ######
       def local_check
-        s3_client.list_objects(bucket: s3params.bucket, prefix: s3params.object_prefix)
-      end
-
-      def get_object(target)
-        opts = { bucket: s3params.bucket, key: archive_object_key }
-        opts[:version_id] = fetch(:s3_archive_object_version_id) if fetch(:s3_archive_object_version_id)
-        s3_client.get_object(opts, target: target)
+        archive_object.check_access!
       end
 
       def remote_check
-        backend.execute :echo, 'check ssh'
+        case strategy
+        when :direct
+          backend.execute :aws, "s3", "ls", ["s3:/", archive_object.bucket, archive_object.key].join("/")
+        when :rsync
+          backend.execute :echo, "ssh connected"
+        end
       end
 
-      def stage
-        stage_lock do
-          archive_dir = File.join(fetch(:s3_archive_local_download_dir), fetch(:stage).to_s)
-          archive_file = File.join(archive_dir, File.basename(archive_object_key))
-          tmp_file = "#{archive_file}.part"
-          etag_file = File.join(archive_dir, ".#{File.basename(archive_object_key)}.etag")
-          fail "#{tmp_file} is found. Another process is running?" if File.exist?(tmp_file)
-          etag = get_object_metadata.tap { |it| fail "No such object: #{current_revision}" if it.nil? }.etag
-
+      def strategy
+        @strategy ||= fetch(:s3_archive_strategy)
+      end
 
-          if [archive_file, etag_file].all? { |f| File.exist?(f) } && File.read(etag_file) == etag
-            backend.info "#{archive_file} (etag:#{etag}) is found. download skipped."
-          else
-            backend.info "Download #{current_revision} to #{archive_file}"
-            mkdir_p(File.dirname(archive_file))
-            File.open(tmp_file, 'w') do |f|
-              get_object(f)
-            end
-            move(tmp_file, archive_file)
-            File.write(etag_file, etag)
-          end
+      def current_revision
+        archive_object.current_revision
+      end
 
-          remove_entry_secure(fetch(:s3_archive_local_cache_dir)) if File.exist? fetch(:s3_archive_local_cache_dir)
-          mkdir_p(fetch(:s3_archive_local_cache_dir))
+      def deploy_to_release_path
+        case strategy
+        when :direct
+          archive_file = remote_cache.target_file
           case archive_file
-          when /\.zip\Z/
-            cmd = "unzip -q -d #{fetch(:s3_archive_local_cache_dir)} #{archive_file}"
-          when /\.tar\.gz\Z|\.tar\.bz2\Z|\.tgz\Z/
-            cmd = "tar xf #{archive_file} -C #{fetch(:s3_archive_local_cache_dir)}"
-          end
-
-          release_lock_on_stage do
-            run_locally do
-              execute cmd
-            end
+          when /\.zip\?.*\Z/
+            backend.execute :unzip, "-q -d", release_path, archive_file
+          when /\.tar\.gz\?.*\Z|\.tar\.bz2\?.*\Z|\.tgz\?.*\Z/
+            backend.execute :tar, "xf", archive_file, "-C", release_path
           end
+        when :rsync
+          link_option = if fetch(:s3_archive_hardlink_release) && backend.test("[ `readlink #{current_path}` != #{release_path} ]")
+                          "--link-dest `readlink #{current_path}`"
+                        end
+          create_release = %[rsync #{fetch(:s3_archive_remote_rsync_copy_option)} #{link_option} "#{rsync_cache_dir}/" "#{release_path}/"]
+          backend.execute create_release
         end
       end
 
-      def cleanup_stage_dir
-        run_locally do
-          archives_dir = File.join(fetch(:s3_archive_local_download_dir), fetch(:stage).to_s)
-          archives = capture(:ls, '-xtr', archives_dir).split
-          if archives.count >= fetch(:keep_releases)
-            to_be_removes = (archives - archives.last(fetch(:keep_releases)))
-            if to_be_removes.any?
-              to_be_removes_str = to_be_removes.map do |file|
-                File.join(archives_dir, file)
-              end.join(' ')
-              execute :rm, to_be_removes_str
-            end
-          end
-        end
+      # for rsync
+      def download_to_local_cache
+        local_cache.download
+        local_cache.extract
       end
 
-      def transfer_sources(dest)
-        fail "#{__method__} must be called in run_locally" unless backend.is_a?(SSHKit::Backend::Local)
-
-        rsync = ['rsync']
-        rsync.concat fetch(:s3_archive_remote_rsync_options, [])
-        rsync << (fetch(:s3_archive_local_cache_dir) + '/')
-
-        if dest.local?
-          rsync << ('--no-compress')
-          rsync << rsync_cache_dir
-        else
-          rsync << "-e 'ssh #{dest.ssh_key_option} #{fetch(:s3_archive_remote_rsync_ssh_options).join(' ')}'"
-          rsync << "#{dest.login_user_at}#{dest.hostname}:#{rsync_cache_dir}"
-        end
-
-        release_lock_on_create do
-          backend.execute(*rsync)
-        end
+      def cleanup_local_cache
+        local_cache.cleanup(keep: fetch(:keep_releases))
       end
 
-      def release
-        link_option = if fetch(:s3_archive_hardlink_release) && backend.test("[ `readlink #{current_path}` != #{release_path} ]")
-                        "--link-dest `readlink #{current_path}`"
-                      end
-        create_release = %[#{fetch(:s3_archive_rsync_copy)} #{link_option} "#{rsync_cache_dir}/" "#{release_path}/"]
-        backend.execute create_release
-      end
+      def transfer_sources(dest)
+        rsync_options = []
+        rsync_options.concat fetch(:s3_archive_remote_rsync_options, [])
+        rsync_options << local_cache.cache_dir + "/"
 
-      def current_revision
-        if fetch(:s3_archive_object_version_id)
-          "#{archive_object_key}?versionid=#{fetch(:s3_archive_object_version_id)}"
+        if dest.local?
+          rsync_options << '--no-compress'
+          rsync_options << rsync_cache_dir
         else
-          archive_object_key
+          rsync_ssh_options = []
+          rsync_ssh_options << dest.ssh_key_option unless dest.ssh_key_option.empty?
+          rsync_ssh_options.concat fetch(:s3_archive_remote_rsync_ssh_options)
+          rsync_options << "-e 'ssh #{rsync_ssh_options.join(' ')}'" unless rsync_ssh_options.empty?
+          rsync_options << "#{dest.login_user_at}#{dest.hostname}:#{rsync_cache_dir}"
         end
-      end
 
-      def archive_object_key
-        @archive_object_key ||=
-          case fetch(:branch, :latest).to_sym
-          when :master, :latest
-            latest_object_key
-          else
-            s3params.object_prefix + fetch(:branch).to_s
-          end
+        backend.execute :rsync, *rsync_options
       end
 
       def rsync_cache_dir
         File.join(deploy_to, fetch(:s3_archive_rsync_cache_dir))
       end
 
-      def s3params
-        @s3params ||= S3Params.new(fetch(:repo_url))
-      end
-
-      def get_object_metadata
-        s3_client.list_object_versions(bucket: s3params.bucket, prefix: archive_object_key).versions.find do |v|
-          if fetch(:s3_archive_object_version_id) then v.version_id == fetch(:s3_archive_object_version_id)
-          else v.is_latest
-          end
-        end
-      end
-
-      def list_all_objects
-        response = s3_client.list_objects(bucket: s3params.bucket, prefix: s3params.object_prefix)
-        response.inject([]) do |objects, page|
-          objects + page.contents
-        end
-      end
-
-      def latest_object_key
-        list_all_objects.sort(&fetch(:s3_archive_sort_proc)).first.key
-      end
-
-      private
-
-      def release_lock_on_stage(&block)
-        release_lock((File::LOCK_EX | File::LOCK_NB), &block) # exclusive lock
-      end
-
-      def release_lock_on_create(&block)
-        release_lock(File::LOCK_SH, &block)
+      # for direct
+      def download_to_shared_path
+        remote_cache.download
       end
 
-      def release_lock(lock_mode, &block)
-        mkdir_p(File.dirname(fetch(:s3_archive_local_cache_dir)))
-        lockfile = "#{fetch(:s3_archive_local_cache_dir)}.#{fetch(:stage)}.release.lock"
-        File.open(lockfile, File::RDONLY | File::CREAT) do |file|
-          if file.flock(lock_mode)
-            block.call
-          else
-            fail ResourceBusyError, "Could not get #{lockfile}"
-          end
-        end
+      def cleanup_shared_path
+        remote_cache.cleanup(keep: fetch(:keep_releases))
       end
 
-      def stage_lock(&block)
-        mkdir_p(File.dirname(fetch(:s3_archive_local_cache_dir)))
-        lockfile = "#{fetch(:s3_archive_local_cache_dir)}.#{fetch(:stage)}.lock"
-        File.open(lockfile, "w") do |file|
-          fail ResourceBusyError, "Could not get #{lockfile}" unless file.flock(File::LOCK_EX | File::LOCK_NB)
-          block.call
-        end
-      ensure
-        rm lockfile if File.exist? lockfile
+      def archive_object
+        @archive_object ||= ArchiveObject.new(repo_url: fetch(:repo_url),
+                                              version_id: fetch(:s3_archive_object_version_id),
+                                              sort_proc: fetch(:s3_archive_sort_proc),
+                                              branch: fetch(:branch),
+                                              client_options: fetch(:s3_archive_client_options))
       end
 
-      def s3_client
-        @s3_client ||= Aws::S3::Client.new(fetch(:s3_archive_client_options))
+      def remote_cache
+        @remote_cache ||= RemoteCache.new(
+          backend,
+          File.join(fetch(:s3_archive_remote_cache_dir), fetch(:stage).to_s),
+          archive_object
+        )
       end
 
-       class LocalExtractor
-        # class ResourceBusyError < StandardError; end
-
-        # include FileUtils
-
-        def stage
-          stage_lock do
-            archive_dir = File.join(fetch(:s3_archive_local_download_dir), fetch(:stage).to_s)
-            archive_file = File.join(archive_dir, File.basename(archive_object_key))
-            tmp_file = "#{archive_file}.part"
-            etag_file = File.join(archive_dir, ".#{File.basename(archive_object_key)}.etag")
-            fail "#{tmp_file} is found. Another process is running?" if File.exist?(tmp_file)
-            etag = get_object_metadata.tap { |it| fail "No such object: #{current_revision}" if it.nil? }.etag
-
-
-            if [archive_file, etag_file].all? { |f| File.exist?(f) } && File.read(etag_file) == etag
-              context.info "#{archive_file} (etag:#{etag}) is found. download skipped."
-            else
-              context.info "Download #{current_revision} to #{archive_file}"
-              mkdir_p(File.dirname(archive_file))
-              File.open(tmp_file, 'w') do |f|
-                get_object(f)
-              end
-              move(tmp_file, archive_file)
-              File.write(etag_file, etag)
-            end
-
-            remove_entry_secure(fetch(:s3_archive_local_cache_dir)) if File.exist? fetch(:s3_archive_local_cache_dir)
-            mkdir_p(fetch(:s3_archive_local_cache_dir))
-            case archive_file
-            when /\.zip\Z/
-              cmd = "unzip -q -d #{fetch(:s3_archive_local_cache_dir)} #{archive_file}"
-            when /\.tar\.gz\Z|\.tar\.bz2\Z|\.tgz\Z/
-              cmd = "tar xf #{archive_file} -C #{fetch(:s3_archive_local_cache_dir)}"
-            end
-
-            release_lock_on_stage do
-              run_locally do
-                execute cmd
-              end
-            end
-          end
-        end
-
-        def stage_lock(&block)
-          mkdir_p(File.dirname(fetch(:s3_archive_local_cache_dir)))
-          lockfile = "#{fetch(:s3_archive_local_cache_dir)}.#{fetch(:stage)}.lock"
-          begin
-            File.open(lockfile, "w") do |file|
-              fail ResourceBusyError, "Could not get #{lockfile}" unless file.flock(File::LOCK_EX | File::LOCK_NB)
-              block.call
-            end
-          ensure
-            rm lockfile if File.exist? lockfile
-          end
-        end
-      end
-
-      class RemoteExtractor
-      end
-
-      class S3Params
-        attr_reader :bucket, :object_prefix
-
-        def initialize(repo_url)
-          uri = URI.parse(repo_url)
-          @bucket = uri.host
-          @object_prefix = uri.path.sub(/\/?\Z/, '/').slice(1..-1) # normalize path
-        end
+      def local_cache
+        @local_cache ||= LocalCache.new(
+          backend,
+          File.join(fetch(:s3_archive_local_download_dir), fetch(:stage).to_s),
+          File.join(fetch(:s3_archive_local_cache_dir), fetch(:stage).to_s),
+          archive_object
+        )
       end
     end
   end

data/lib/capistrano/scm/tasks/s3_archive.rake

@@ -7,51 +7,55 @@ namespace :s3_archive do
       plugin.local_check
     end
 
-    on release_roles :all do
+    on release_roles(:all) do
       plugin.remote_check
     end
   end
 
-  desc 'Extruct and stage the S3 archive in a stage directory'
-  task :stage do
-    if fetch(:skip_staging, false)
-      info "Skip extracting and staging."
-      next
-    end
-
-    run_locally do
-      plugin.stage
+  desc 'Deploy to release_path'
+  task create_release: :stage do
+    on release_roles(:all) do
+      execute :mkdir, '-p', release_path
+      plugin.deploy_to_release_path
     end
   end
 
-  after :stage, :cleanup_stage_dir do
-    run_locally do
-      plugin.cleanup_stage_dir
-    end
+  desc 'Determine the revision that will be deployed'
+  task :set_current_revision do
+    set :current_revision, plugin.current_revision
   end
 
-  desc 'Copy repo to releases'
-  task create_release: :stage do
-    on release_roles(:all), fetch(:s3_archive_remote_rsync_runner_options) do |server|
-      test "[ -e #{plugin.rsync_cache_dir} ]" # implicit initialize for 'server'
+  desc 'Stage the S3 archive to cache directory'
+  task :stage do
+    case plugin.strategy
+    when :direct
+      on release_roles(:all) do
+        plugin.download_to_shared_path
+      end
+    when :rsync
       run_locally do
-        plugin.transfer_sources(server)
+        plugin.download_to_local_cache unless fetch(:s3_archive_skip_download)
       end
-    end
-
-    on release_roles(:all) do
-      execute :mkdir, '-p', release_path
-      plugin.release
+      on release_roles(:all), fetch(:s3_archive_remote_rsync_runner_options) do |server|
+        test "[ -e #{plugin.rsync_cache_dir} ]" # implicit initialize for 'server'
+        run_locally { plugin.transfer_sources(server) }
+      end
+    else
+      error "Invalid stragegy #{plugin.strategy} of SCM::S3Archive"
+      exit 1
     end
   end
 
-  desc 'Determine the revision that will be deployed'
-  task :set_current_revision do
-    set :current_revision, plugin.current_revision
+  after :stage, :cleanup_stage_dir do
+    case plugin.strategy
+    when :direct
+      on release_roles(:all) do
+        plugin.cleanup_shared_path
+      end
+    when :rsync
+      run_locally do
+        plugin.cleanup_local_cache
+      end
+    end
   end
 end unless Rake::Task.task_defined?("s3_archive:check")
-
-task :deploy_only do
-  set :skip_staging, true
-  invoke :deploy
-end