capistrano-net_storage 0.3.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 507fe8707e3df873b757a6a44e830852f67d4444
-  data.tar.gz: 15d1fa62fb07eabb8c40fbe22cb5007f9bef445f
+SHA256:
+  metadata.gz: f402751b93d15bbc8660b9669a18977ac69d67381d45c0316afd89d185f750fe
+  data.tar.gz: 38905906ddf2c6aae2706679eb907b6970c1099e058eb8f8c676932656cc3f7d
 SHA512:
-  metadata.gz: 06d7db8a593c0badde8a5f16f833a3f42f4ab1add488fcb88b1f6bb51f48d8d65a4feac9c14b833c6393de6d0d13225d54752d33afe244e0fb6c3c6033a5ac36
-  data.tar.gz: d17853a6e94973c78e41f1ac807e635da7ca33d50bd131559a443e75ef6fabc0cd82d14e8344f67bdecaeab5f50196f448a969ccb37a3b5389495926bd010e28
+  metadata.gz: 3c61f03392a39f8f69c824fc196ef3e056c206e1c00ecce038e73f18b8782074d4ef5895fbf65d4c5e42666950ab0b0c53b99ceeeeb5d428122c92caa1dc8373
+  data.tar.gz: cd42fc4f35a13075920dff572727a93e4f9db1d28249833d0af85a5edc82d661303a2595511ab2b7067f5a3939cead39d5a5a68ffe7a2577799ea94f03b9c099

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,7 @@
+### Summary
+
+### Other Information
+
+### Checklist
+
+* [ ] By placing an "x" in the box, I hereby understand, accept and agree to be bound by the terms and conditions of the [Contribution License Agreement](https://dena.github.io/cla/).

data/.travis.yml

@@ -1,13 +1,11 @@
 sudo: false
 language: ruby
 rvm:
-- 2.0.0
-- 2.1.8
-- 2.2.0
-- 2.3.5
-- 2.4.2
+- 2.6.3
+- 2.5.5
+- 2.4.6
 
-before_install: gem install bundler -v 1.14.6
+before_install: gem install bundler
 install: bundle install
 script: bundle exec rake spec
 notifications:

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 0.4.0 (2022/05/17)
+
+Fix Bug:
+
+- Fix a cleanup issue reported at #9 (#12) @naro143
+  - Before:
+    - Archives such as `.tar.gz` and `.zip` are handled in `releases` directory of each server.
+    - Incomplete deployment leads to uncollected garbage directory under `releases`.
+  - After:
+    - Archives are handled within `#{deploy_to}/net_storage_archives` directory of each server.
+    - NetStorage cleanup `net_storage_archives` directory just as Capistrano do in `releases` directory.
+    - One successful deployment can cleanup old archives and there is no more Capistrano warnings.
+  - This includes the same cleanup policy for `local_releases_path` and `local_archives_path` of NetStorage.
+
+Changes:
+
+- Change task name `net_storage:cleanup_remote_release` to `net_storage:cleanup_archives_on_remote_storage`
+
 ## 0.3.2 (2017/11/7)
 
 Fix Bug:

data/README.md

@@ -49,15 +49,21 @@ Set Capistrano variables by `set name, value`.
  `:scm`  | `nil` | Set `:net_storage` for capistrano before v3.7
  `:branch` | `master` | Target branch of SCM to release
  `:keep_releases` | `5` | Numbers to keep released versions
- `:net_storage_transport` | `nil` | Transport class for _remote storage_
  `:net_storage_archiver` | `Capistrano::NetStorage::Archiver::Zip` | Archiver class
  `:net_storage_scm` | `Capistrano::NetStorage::SCM::Git` | Internal scm class for application repository
- `:net_storage_with_bundle` | `false` | Do `bundle install` when creating archive
+ `:net_storage_transport` | `nil` | Transport class for _remote storage_
+ `:net_storage_archive_on_missing` | `true` | If `true`, create and upload archive only when target archive is missing on remote storage
  `:net_storage_config_files` | `nil` | Files to sync `config/` directory on target servers' application directory
  `:net_storage_max_parallels` | number of servers | Max concurrency for remote tasks
- `:net_storage_archive_on_missing` | `true` | If `true`, create and upload archive only when target archive is missing on remote storage
+ `:net_storage_rsync_options` | `#{ssh_options}` | SSH options for rsync command to sync configs
  `:net_storage_upload_files_by_rsync` | `false` | Use rsync(1) to deploy config files
+ `:net_storage_with_bundle` | `false` | Do `bundle install` when creating archive
  `:net_storage_local_base_path` | `.local_repo` | Base directory on deploy server
+ `:net_storage_local_mirror_path` | `#{net_storage_local_base_path}/mirror` | Path to clone repository on deploy server
+ `:net_storage_local_releases_path` | `#{net_storage_local_base_path}/releases` | Path to keep release directories on deploy server
+ `:net_storage_local_bundle_path` | `#{net_storage_local_base_path}/bundle` | Shared directory to install gems on deploy server
+ `:net_storage_local_archives_path` | `#{net_storage_local_base_path}/archives` | Archive directories on deploy server
+ `:net_storage_archives_path` | `#{deploy_to}/net_storage_archives` | Archive directories on application server
 
 ### Transport Plugins
 

data/capistrano-net_storage.gemspec

@@ -21,7 +21,6 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = '>= 2.0'
 
   spec.add_runtime_dependency 'capistrano', '>= 3.3.3'
-  spec.add_runtime_dependency 'parallel'
   spec.add_runtime_dependency 'bundler'
 
   spec.add_development_dependency 'rake'

data/lib/capistrano/net_storage/archiver/base.rb

@@ -16,8 +16,7 @@ module Capistrano
           raise NotImplementedError
         end
 
-        # Extract archive.
-        # Archive file should be cleaned up after extracted.
+        # Extract archive
         # @abstract
         def extract
           raise NotImplementedError

data/lib/capistrano/net_storage/archiver/tar_gzip.rb

@@ -23,10 +23,10 @@ class Capistrano::NetStorage::Archiver::TarGzip < Capistrano::NetStorage::Archiv
   def extract
     c = config
     on c.servers, in: :groups, limit: c.max_parallels do
+      execute :mkdir, '-p', c.archives_path
       execute :mkdir, '-p', release_path
       within release_path do
         execute :tar, 'xzf', c.archive_path
-        execute :rm, '-f', c.archive_path
       end
     end
   end

data/lib/capistrano/net_storage/archiver/zip.rb

@@ -23,8 +23,11 @@ class Capistrano::NetStorage::Archiver::Zip < Capistrano::NetStorage::Archiver::
   def extract
     c = config
     on c.servers, in: :groups, limit: c.max_parallels do
-      execute :unzip, c.archive_path, '-d', release_path
-      execute :rm, '-f', c.archive_path
+      execute :mkdir, '-p', c.archives_path
+      execute :mkdir, '-p', release_path
+      within release_path do
+        execute :unzip, c.archive_path
+      end
     end
   end
 end

data/lib/capistrano/net_storage/cleaner.rb

@@ -6,22 +6,95 @@ module Capistrano
     class Cleaner
       include Capistrano::NetStorage::Utils
 
-      # Clean up local release directories and archives.
+      # Clean up old release directories on local.
       # Assumes they are under +config.local_releases_path+
       # @see Capistrano::NetStorage::Config#local_releases_path
-      def cleanup_local_release
+      def cleanup_local_releases
         c = config
         run_locally do
-          releases = capture(:ls, '-xtr', c.local_releases_path).split
-          # Contains archive files and extracted directories
-          if releases.count > fetch(:keep_releases) * 2
-            info "Keeping #{fetch(:keep_releases)} * 2 of #{releases.count} local releases"
-            olds_str = (releases - releases.last(fetch(:keep_releases) * 2)).map do |file|
-              c.local_releases_path.join(file)
-            end.join(' ')
-            execute :rm, '-rf', olds_str
+          contents = capture(:ls, '-x', c.local_releases_path).split
+          # reference
+          # https://github.com/capistrano/capistrano/blob/cc4f31fdfcb4a21c569422a95868d0bb52844c75/lib/capistrano/tasks/deploy.rake#L152
+          releases, invalid = contents.partition { |e| /^\d{14}$/ =~ e }
+
+          if invalid.any?
+            warn "Invalid contents in #{c.local_releases_path} on local:\n#{invalid.join("\n")}"
+          end
+
+          if releases.count > fetch(:keep_releases)
+            info "Keeping #{fetch(:keep_releases)} of #{releases.count} releases on local"
+            old_releases = (releases - releases.last(fetch(:keep_releases))).map do |release|
+              c.local_releases_path.join(release).to_s
+            end
+            execute :rm, '-rf', *old_releases
+          else
+            info "No old releases (keeping newest #{fetch(:keep_releases)}) in #{c.local_releases_path} on local"
+          end
+        end
+      end
+
+      # Clean up old archive files on local.
+      # Assumes they are under +config.local_archives_path+
+      # @see Capistrano::NetStorage::Config#local_archives_path
+      def cleanup_local_archives
+        c = config
+        run_locally do
+          contents = capture(:ls, '-x', c.local_archives_path).split
+          archives, invalid = contents.partition { |e| /^\d{14}\.#{Regexp.escape(c.archive_suffix)}$/ =~ e }
+
+          if invalid.any?
+            warn "Invalid contents in #{c.local_archives_path} on local:\n#{invalid.join("\n")}"
+          end
+
+          if archives.count > fetch(:keep_releases)
+            info "Keeping #{fetch(:keep_releases)} of #{archives.count} archives on local"
+            old_archives = (archives - archives.last(fetch(:keep_releases))).map do |archive|
+              c.local_archives_path.join(archive).to_s
+            end
+            execute :rm, '-f', *old_archives
+          else
+            info "No old archives (keeping newest #{fetch(:keep_releases)}) in #{c.local_archives_path} on local"
+          end
+        end
+      end
+
+      # Clean up old archive files on remote servers.
+      # Assumes they are under +config.archives_path+
+      # @see Capistrano::NetStorage::Config#archives_path
+      def cleanup_archives
+        c = config
+        on c.servers, in: :groups, limit: c.max_parallels do |host|
+          contents = capture(:ls, '-x', c.archives_path).split
+          archives, invalid = contents.partition { |e| /^\d{14}\.#{Regexp.escape(c.archive_suffix)}$/ =~ e }
+
+          if invalid.any?
+            warn "Invalid contents in #{c.archives_path} on #{host}:\n#{invalid.join("\n")}"
+          end
+
+          if archives.count > fetch(:keep_releases)
+            info "Keeping #{fetch(:keep_releases)} of #{archives.count} archives on #{host}"
+            old_archives = (archives - archives.last(fetch(:keep_releases))).map do |archive|
+              c.archives_path.join(archive).to_s
+            end
+
+            if test("[ -d #{current_path} ]")
+              current_release = capture(:readlink, current_path).to_s
+              current_release_archive = c.archives_path.join("#{File.basename(current_release)}.#{c.archive_suffix}")
+              if old_archives.include?(current_release_archive)
+                warn "Current release archive was marked for being removed but it's going to be skipped on #{host}"
+                old_archives.delete(current_release_archive)
+              end
+            else
+              debug "There is no current release present on #{host}"
+            end
+
+            if old_archives.any?
+              old_archives.each_slice(100) do |old_archives_batch|
+                execute :rm, '-f', *old_archives_batch
+              end
+            end
           else
-            info "No old local releases in #{c.local_releases_path}"
+            info "No old archives (keeping newest #{fetch(:keep_releases)}) in #{c.archives_path} on #{host}"
           end
         end
       end

data/lib/capistrano/net_storage/config.rb

@@ -53,7 +53,7 @@ module Capistrano
         @skip_bundle
       end
 
-      # If +true+, create archive ONLY when it's not found on Network Storage.
+      # If +true+, create archive ONLY when it's not found on remote storage.
       # Otherwise, create archive ALWAYS.
       # Defaults to +true+
       def archive_on_missing?
@@ -83,56 +83,63 @@ module Capistrano
       # Path settings
       #
 
-      # Path of base directory on localhost
+      # Path of base directory on local
       # @return [Pathname]
       def local_base_path
         @local_base_path ||= pathname(fetch(:net_storage_local_base_path, "#{Dir.pwd}/.local_repo"))
       end
 
-      # Path to clone repository on localhost
+      # Path to clone repository on local
       # @return [Pathname]
       def local_mirror_path
         @local_mirror_path ||= pathname(fetch(:net_storage_local_mirror_path))
         @local_mirror_path ||= local_base_path.join('mirror')
       end
 
-      # Path to keep release directories and archives on localhost
+      # Path to keep release directories on local
       # @return [Pathname]
       def local_releases_path
         @local_releases_path ||= pathname(fetch(:net_storage_local_releases_path))
         @local_releases_path ||= local_base_path.join('releases')
       end
 
-      # Path to take a snapshot of repository for release
+      # Path to take a snapshot of repository for release on local
       # @return [Pathname]
       def local_release_path
-        @local_release_path ||= pathname(fetch(:net_storage_local_release_path))
         @local_release_path ||= local_releases_path.join(release_timestamp)
       end
 
-      # Shared directory to install gems
+      # Shared directory to install gems on local
       # @return [Pathname]
       def local_bundle_path
         @local_bundle_path ||= pathname(fetch(:net_storage_local_bundle_path))
         @local_bundle_path ||= local_base_path.join('bundle')
       end
 
-      # Destination path to archive application on localhost
+      # Path of archive directories on local
+      # @return [Pathname]
+      def local_archives_path
+        @local_archives_path ||= pathname(fetch(:net_storage_local_archives_path))
+        @local_archives_path ||= local_base_path.join('archives')
+      end
+
+      # Destination path to archive application on local
       # @return [Pathname]
       def local_archive_path
-        @local_archive_path ||= pathname(fetch(:net_storage_local_archive_path))
-        @local_archive_path ||= pathname("#{local_release_path}.#{archive_suffix}")
+        @local_archive_path ||= local_archives_path.join("#{release_timestamp}.#{archive_suffix}")
       end
 
-      # Path of archive file to be downloaded on servers
+      # Path of archive directories on remote servers
+      # @return [Pathname]
+      def archives_path
+        @archives_path ||= pathname(fetch(:net_storage_archives_path))
+        @archives_path ||= deploy_path.join('net_storage_archives')
+      end
+
+      # Path of archive file to be downloaded on remote servers
       # @return [Pathname]
       def archive_path
-        @archive_path ||= pathname(fetch(:net_storage_archive_path))
-        @archive_path ||= begin
-          # Set release_timestamp if not set
-          fetch(:release_path, set_release_path)
-          pathname("#{release_path}.#{archive_suffix}")
-        end
+        @archive_path ||= archives_path.join("#{release_timestamp}.#{archive_suffix}")
       end
 
       # Suffix of archive file

data/lib/capistrano/net_storage/transport/base.rb

@@ -1,7 +1,7 @@
 module Capistrano
   module NetStorage
     module Transport
-      # Abstract class to transport archive from/to Network Storage
+      # Abstract class to transport archive from/to remote storage
       # @abstract
       class Base
         # Check prerequisites for transport

data/lib/capistrano/net_storage/utils.rb

@@ -1,5 +1,3 @@
-require 'parallel'
-
 require 'capistrano/net_storage/base'
 
 module Capistrano
@@ -17,17 +15,19 @@ module Capistrano
       def upload_files(files, dest_dir)
         c = config
         hosts = ::Capistrano::Configuration.env.filter(c.servers)
-        if c.upload_files_by_rsync?
-          Parallel.each(hosts, in_threads: c.max_parallels) do |host|
-            ssh = build_ssh_command(host)
+
+        # FIXME: This is a very workaround to architectural issue. Do not copy.
+        build_rsh_option = -> (host) {
+          build_ssh_command(host)
+        }
+
+        on hosts, in: :groups, limit: c.max_parallels do |host|
+          if c.upload_files_by_rsync?
+            rsh_option = build_rsh_option.call(host)
             run_locally do
-              files.each do |src|
-                execute :rsync, "-az --rsh='#{ssh}' #{src} #{host}:#{dest_dir}"
-              end
+              execute :rsync, "-az --rsh='#{rsh_option}' #{files.join(' ')} #{host.hostname}:#{dest_dir}"
             end
-          end
-        else
-          on c.servers, in: :groups, limit: c.max_parallels do
+          else
             files.each do |src|
               upload! src, dest_dir
             end

data/lib/capistrano/net_storage/version.rb

@@ -1,5 +1,5 @@
 module Capistrano
   module NetStorage
-    VERSION = '0.3.2'
+    VERSION = '0.5.0'
   end
 end

data/lib/capistrano/tasks/net_storage.rake

@@ -43,19 +43,31 @@ namespace :net_storage do
   end
   after 'net_storage:create_release', 'net_storage:sync_config'
 
-  desc 'Clean up local old archives and snapshots'
-  task :cleanup_local_release do
-    Capistrano::NetStorage.cleaner.cleanup_local_release
+  desc 'Clean up old release directories on local'
+  task :cleanup_local_releases do
+    Capistrano::NetStorage.cleaner.cleanup_local_releases
   end
-  after 'deploy:cleanup', 'net_storage:cleanup_local_release'
+  after 'deploy:cleanup', 'net_storage:cleanup_local_releases'
 
-  desc 'Clean up old archives on remote storage'
-  task :cleanup_remote_release do
+  desc 'Clean up old archive files on remote storage'
+  task :cleanup_archives_on_remote_storage do
     transport = Capistrano::NetStorage.transport
     next unless transport.respond_to?(:cleanup)
     transport.cleanup
   end
-  after 'net_storage:cleanup_local_release', 'net_storage:cleanup_remote_release'
+  after 'deploy:cleanup', 'net_storage:cleanup_archives_on_remote_storage'
+
+  desc 'Clean up old archive files on remote servers'
+  task :cleanup_archives do
+    Capistrano::NetStorage.cleaner.cleanup_archives
+  end
+  after 'deploy:cleanup', 'net_storage:cleanup_archives'
+
+  desc 'Clean up old archive files on local'
+  task :cleanup_local_archives do
+    Capistrano::NetStorage.cleaner.cleanup_local_archives
+  end
+  after 'deploy:cleanup', 'net_storage:cleanup_local_archives'
 
   task prepare_archive: %i(net_storage:scm:update net_storage:check:bundler) do
     config = Capistrano::NetStorage.config
@@ -103,6 +115,7 @@ namespace :net_storage do
         config.local_base_path,
         config.local_mirror_path,
         config.local_releases_path,
+        config.local_archives_path,
       ]
       dirs << config.local_bundle_path unless config.skip_bundle?
       run_locally do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: capistrano-net_storage
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.5.0
 platform: ruby
 authors:
 - progrhyme
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2023-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capistrano
@@ -24,20 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.3.3
-- !ruby/object:Gem::Dependency
-  name: parallel
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -97,11 +83,12 @@ dependencies:
 description: |2
       A Capistrano SCM Plugin to deploy application via remove storage.
       Logically, this enables O(1) deployment.
-email: 
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/PULL_REQUEST_TEMPLATE.md"
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
@@ -134,7 +121,7 @@ homepage: https://github.com/DeNADev/capistrano-net_storage
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -149,9 +136,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubygems_version: 3.4.13
+signing_key:
 specification_version: 4
 summary: Capistrano SCM Plugin for fast deployment via remote storage
 test_files: []