capistrano-net_storage 0.4.0 → 1.0.0

data/lib/capistrano/net_storage/config.rb

@@ -1,169 +1,177 @@
-require 'capistrano/net_storage/error'
-require 'capistrano/net_storage/bundler'
-require 'capistrano/net_storage/archiver/zip'
-require 'capistrano/net_storage/scm/git'
+require 'pathname'
+
+require 'capistrano/net_storage/cleaner'
+require 'capistrano/net_storage/bundler/default'
+require 'capistrano/net_storage/bundler/null'
+require 'capistrano/net_storage/config_uploader'
 
 module Capistrano
   module NetStorage
     class Config
-      def executor_class(type)
-        @executor_classes ||= {}
-        @executor_classes[type] ||= fetch(:"net_storage_#{type}")
-        @executor_classes[type] ||= begin
-          case type
-          when :archiver
-            Capistrano::NetStorage::Archiver::Zip
-          when :scm
-            Capistrano::NetStorage::SCM::Git
-          when :cleaner
-            Capistrano::NetStorage::Cleaner
-          when :bundler
-            Capistrano::NetStorage::Bundler
-          when :transport
-            msg = 'You have to set :net_storage_transport because no default transport class!'
-            raise Capistrano::NetStorage::Error, msg
-          else
-            raise "Unknown type! #{type}"
-          end
-        end
+
+      # Settings for delegate classes
+
+      def transport_class
+        fetch(:net_storage_transport)
       end
 
-      # Servers to deploy
-      def servers
-        fetch(:net_storage_servers, -> { release_roles(:all) })
+      def archiver_class
+        fetch(:net_storage_archiver)
       end
 
-      def max_parallels
-        @max_parallels ||= fetch(:net_storage_max_parallels, servers.size)
+      def scm_class
+        fetch(:net_storage_scm)
       end
 
-      # Application configuration files to be deployed with
-      # @return [Array<String, Pathname>]
-      def config_files
-        @config_files ||= fetch(:net_storage_config_files)
+      def cleaner_class
+        Capistrano::NetStorage::Cleaner
       end
 
-      # If +true+, skip to bundle gems bundled with target app.
-      # Defaults to +true+
-      def skip_bundle?
-        @has_checked_skip_bundle ||= begin
-          @skip_bundle = !fetch(:net_storage_with_bundle)
-          true
+      def bundler_class
+        if skip_bundle?
+          Capistrano::NetStorage::Bundler::Null
+        else
+          Capistrano::NetStorage::Bundler::Default
         end
-        @skip_bundle
       end
 
-      # If +true+, create archive ONLY when it's not found on remote storage.
-      # Otherwise, create archive ALWAYS.
-      # Defaults to +true+
-      def archive_on_missing?
-        @has_checked_archive_on_missing ||= begin
-          @archive_on_missing = fetch(:net_storage_archive_on_missing, true)
-          true
-        end
-        @archive_on_missing
+      def config_uploader_class
+        Capistrano::NetStorage::ConfigUploader
+      end
+
+      # Settings for syncing config
+
+      def config_files
+        fetch(:net_storage_config_files)
       end
 
-      # If +true+, use +rsync+ to sync config files.
-      # Otherwise, use +upload!+ by sshkit.
-      # Defaults to +false+
-      # @see #rsync_options
       def upload_files_by_rsync?
-        @upload_files_by_rsync ||= fetch(:net_storage_upload_files_by_rsync, false)
+        fetch(:net_storage_upload_files_by_rsync)
       end
 
-      # You can set +:user+, +:keys+, +:port+ as ssh options for +rsync+ command to sync configs
-      # when +:net_storage_upload_files_by_rsync+ is set +true+.
-      # @see #upload_files_by_rsync?
       def rsync_options
-        @rsync_options ||= fetch(:net_storage_rsync_options, fetch(:ssh_options, {}))
+        fetch(:net_storage_rsync_options)
+      end
+
+      # Settings for tuning performance
+
+      def max_parallels
+        fetch(:net_storage_max_parallels)
+      end
+
+      def reuse_archive?
+        fetch(:net_storage_reuse_archive)
+      end
+
+      def keep_remote_archives
+        fetch(:net_storage_keep_remote_archives)
+      end
+
+      # Settings for behavioral changes
+
+      def skip_bundle?
+        fetch(:net_storage_skip_bundle)
+      end
+
+      def multi_app_mode?
+        fetch(:net_storage_multi_app_mode)
       end
 
-      #
       # Path settings
-      #
 
-      # Path of base directory on local
+      # Path to application of remote release_path
+      # @return [Pathname]
+      def release_app_path
+        if multi_app_mode?
+          release_path.join(fetch(:application))
+        else
+          release_path
+        end
+      end
+
+      # Path to base directory on local
       # @return [Pathname]
       def local_base_path
-        @local_base_path ||= pathname(fetch(:net_storage_local_base_path, "#{Dir.pwd}/.local_repo"))
+        Pathname.new(fetch(:net_storage_local_base_path))
       end
 
-      # Path to clone repository on local
+      # Path to a mirror repository on local
       # @return [Pathname]
       def local_mirror_path
-        @local_mirror_path ||= pathname(fetch(:net_storage_local_mirror_path))
-        @local_mirror_path ||= local_base_path.join('mirror')
+        local_base_path.join('mirror')
       end
 
       # Path to keep release directories on local
       # @return [Pathname]
       def local_releases_path
-        @local_releases_path ||= pathname(fetch(:net_storage_local_releases_path))
-        @local_releases_path ||= local_base_path.join('releases')
+        local_base_path.join('releases')
       end
 
-      # Path to take a snapshot of repository for release on local
+      # Path to local release directory, where a release is to be prepared
       # @return [Pathname]
       def local_release_path
-        @local_release_path ||= local_releases_path.join(release_timestamp)
+        local_releases_path.join(release_timestamp)
       end
 
-      # Shared directory to install gems on local
+      # Path to application of local release_path
+      # @return [Pathname]
+      def local_release_app_path
+        if multi_app_mode?
+          local_release_path.join(fetch(:application))
+        else
+          local_release_path
+        end
+      end
+
+      # Shared cache directory to speed up installing gems on local
       # @return [Pathname]
       def local_bundle_path
-        @local_bundle_path ||= pathname(fetch(:net_storage_local_bundle_path))
-        @local_bundle_path ||= local_base_path.join('bundle')
+        local_base_path.join('bundle')
       end
 
       # Path of archive directories on local
       # @return [Pathname]
       def local_archives_path
-        @local_archives_path ||= pathname(fetch(:net_storage_local_archives_path))
-        @local_archives_path ||= local_base_path.join('archives')
+        local_base_path.join('archives')
       end
 
       # Destination path to archive application on local
       # @return [Pathname]
       def local_archive_path
-        @local_archive_path ||= local_archives_path.join("#{release_timestamp}.#{archive_suffix}")
+        local_archives_path.join("#{release_timestamp}.#{archive_file_extension}")
       end
 
       # Path of archive directories on remote servers
       # @return [Pathname]
       def archives_path
-        @archives_path ||= pathname(fetch(:net_storage_archives_path))
-        @archives_path ||= deploy_path.join('net_storage_archives')
+        Pathname.new(fetch(:net_storage_archives_path))
       end
 
       # Path of archive file to be downloaded on remote servers
       # @return [Pathname]
       def archive_path
-        @archive_path ||= archives_path.join("#{release_timestamp}.#{archive_suffix}")
+        archives_path.join("#{release_timestamp}.#{archive_file_extension}")
       end
 
       # Suffix of archive file
       # @return [String]
-      def archive_suffix
-        case Capistrano::NetStorage.archiver
-        when Capistrano::NetStorage::Archiver::Zip
-          'zip'
-        when Capistrano::NetStorage::Archiver::TarGzip
-          'tar.gz'
-        else
-          'archive'
-        end
+      def archive_file_extension
+        archiver_class.file_extension
       end
 
-      private
+      def archive_suffix
+        warn <<~WARN
+          ######### DEPRECATION WARNING #########
 
-      def pathname(path)
-        case path
-        when String
-          Pathname.new(path)
-        else
-          path
-        end
+          `Capistrano::NetStorage.config.archive_suffix` is no longer available
+          at #{caller[0]}
+
+          Use following method instead.
+          `Capistrano::NetStorage.config.archive_file_extension`
+
+        WARN
+
+        archive_file_extension
       end
     end
   end

data/lib/capistrano/net_storage/config_uploader.rb

@@ -0,0 +1,58 @@
+module Capistrano
+  module NetStorage
+    class ConfigUploader
+      # These values are intentionally separated from config and fixed.
+      # If you have trouble with these defaults for 10^2 ~ 10^3 servers, please contact us on GitHub.
+      # Ideally, you can upload files to 10000 servers in 50 seconds. (2.0 second jitter + 0.5 second execution time)
+      MAX_PARALLEL_TO_UPLOAD = 500
+      JITTER_DURATION_TO_UPLOAD = 2.0
+
+      # Check for prerequisites.
+      # Same interface to delegated class
+      def check
+      end
+
+      def upload_config_files
+        config = Capistrano::NetStorage.config
+
+        on release_roles(:all), in: :groups, limit: MAX_PARALLEL_TO_UPLOAD do |host|
+          if config.upload_files_by_rsync?
+            rsh_option = Capistrano::NetStorage::ConfigUploader.build_ssh_command(host)
+            run_locally do
+              sleep Random.rand(JITTER_DURATION_TO_UPLOAD)
+              execute :rsync, '-az', "--rsh='#{rsh_option}'", *config.config_files, "#{host.hostname}:#{config.release_app_path.join('config')}"
+            end
+          else
+            # slow and not recommended.
+            files.each do |src|
+              upload! src, dest_dir
+            end
+          end
+        end
+      end
+
+      # Build ssh command with options for rsync
+      def self.build_ssh_command(host)
+        user_opt    = ''
+        key_opt     = ''
+        port_opt    = ''
+        ssh_options = Capistrano::NetStorage.config.rsync_options
+
+        if user = host.user || ssh_options[:user]
+          user_opt = " -l #{user}"
+        end
+
+        if keys = (host.keys.empty? ? ssh_options[:keys] : host.keys)
+          keys    = keys.is_a?(Array) ? keys : [keys]
+          key_opt = keys.map { |key| " -i #{key}" }.join('')
+        end
+
+        if port = host.port || ssh_options[:port]
+          port_opt = " -p #{port}"
+        end
+
+        "ssh#{user_opt}#{key_opt}#{port_opt}"
+      end
+    end
+  end
+end

data/lib/capistrano/net_storage/plugin.rb

@@ -1,14 +1,52 @@
+require 'pathname'
+
 require 'capistrano/scm/plugin'
-require 'capistrano/net_storage'
+
+require 'capistrano/net_storage/archiver/tar_gzip'
+require 'capistrano/net_storage/scm/git'
 
 module Capistrano
   module NetStorage
     class Plugin < ::Capistrano::SCM::Plugin
+      # See README.md for details of settings
+      def set_defaults
+        set_if_empty :net_storage_transport, -> { raise ArgumentError, 'You have to `set(:net_storage_transport, Capistrano::NetStorage::S3)` # or your custom class' }
+        set_if_empty :net_storage_archiver, Capistrano::NetStorage::Archiver::TarGzip
+        set_if_empty :net_storage_scm, Capistrano::NetStorage::SCM::Git
+
+        set_if_empty :net_storage_config_files, []
+        set_if_empty :net_storage_upload_files_by_rsync, true
+        set_if_empty :net_storage_rsync_options, -> { fetch(:ssh_options, {}) }
+
+        set_if_empty :net_storage_max_parallels, 1000
+        set_if_empty :net_storage_reuse_archive, true
+
+        set_if_empty :net_storage_local_base_path, Pathname.new("#{Dir.pwd}/.local_net_storage")
+        set_if_empty :net_storage_archives_path, -> { deploy_path.join('net_storage_archives') }
+
+        set_if_empty :net_storage_skip_bundle, false
+        set_if_empty :net_storage_multi_app_mode, false
+
+        set_if_empty :net_storage_keep_remote_archives, 10
+      end
+
+      def define_tasks
+        eval_rakefile File.expand_path("../tasks/net_storage.rake", __dir__)
+      end
+
       def register_hooks
         after  'deploy:new_release_path', 'net_storage:create_release'
         before 'deploy:check', 'net_storage:check'
-        before 'deploy:set_current_revision', 'net_storage:set_current_revision'
+        after 'deploy:cleanup', 'net_storage:cleanup'
       end
     end
   end
 end
+
+# initialization code to access global settings via Capistrano::NetStorage
+
+require 'capistrano/net_storage'
+require 'capistrano/net_storage/config'
+
+config = Capistrano::NetStorage::Config.new
+Capistrano::NetStorage.setup!(config: Capistrano::NetStorage::Config.new)

data/lib/capistrano/net_storage/scm/base.rb

@@ -1,47 +1,26 @@
-require 'capistrano/net_storage/utils'
-
 module Capistrano
   module NetStorage
     module SCM
-      # Base internal SCM class of Capistrano::Netstrage
-      # @abstract
+      # Abstract class to handle SCM between local_mirror_path and local_release_path
       class Base
-        include Capistrano::NetStorage::Utils
-
-        # Check SCM prerequisites
-        # @abstract
         def check
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to check prerequisites for SCM"
         end
 
-        # Clone repository to local
-        # @abstract
         def clone
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to clone repository to `Capistrano::NetStorage.config.local_mirror_path`"
         end
 
-        # Update local repository
-        # @abstract
         def update
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to update repository in `Capistrano::NetStorage.config.local_mirror_path`"
         end
 
-        # Set current revision to be deployed of the repository
-        # @abstract
         def set_current_revision
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to set current revision by `set :current_revision, revision`"
         end
 
-        # Prepare snapshot of repository to be archived for release
-        # @abstract
         def prepare_archive
-          raise NotImplementedError
-        end
-
-        # Copy local config files to servers
-        def sync_config
-          return unless config.config_files
-          upload_files(config.config_files, release_path.join('config'))
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to extract and prepare release into `Capistrano::NetStorage.config.local_release_path`"
         end
       end
     end

data/lib/capistrano/net_storage/scm/git.rb

@@ -1,6 +1,5 @@
 require 'capistrano/net_storage/scm/base'
 
-# Internal SCM class for Git repository
 class Capistrano::NetStorage::SCM::Git < Capistrano::NetStorage::SCM::Base
   def check
     run_locally do
@@ -9,51 +8,54 @@ class Capistrano::NetStorage::SCM::Git < Capistrano::NetStorage::SCM::Base
   end
 
   def clone
-    c = config
+    config = Capistrano::NetStorage.config
+
     run_locally do
-      if File.exist?("#{c.local_mirror_path}/HEAD")
-        info t(:mirror_exists, at: c.local_mirror_path)
+      if test "[ -f #{config.local_mirror_path}/HEAD ]"
+        info t(:mirror_exists, at: config.local_mirror_path)
       else
-        execute :git, :clone, '--mirror', repo_url, c.local_mirror_path
+        execute :git, :clone, '--mirror', repo_url, config.local_mirror_path
       end
     end
   end
 
   def update
-    c = config
+    config = Capistrano::NetStorage.config
+
     run_locally do
-      within c.local_mirror_path do
+      within config.local_mirror_path do
         execute :git, :remote, :update
       end
     end
   end
 
   def set_current_revision
-    return if fetch(:current_revision)
-    c = config
+    config = Capistrano::NetStorage.config
+
     run_locally do
-      within c.local_mirror_path do
+      within config.local_mirror_path do
         set :current_revision, capture(:git, "rev-list --max-count=1 #{fetch(:branch)}")
       end
     end
   end
 
   def prepare_archive
-    c = config
+    config = Capistrano::NetStorage.config
+
     run_locally do
-      execute :mkdir, '-p', c.local_release_path
+      execute :mkdir, '-p', config.local_release_path
 
-      within c.local_mirror_path do
+      within config.local_mirror_path do
         if tree = fetch(:repo_tree)
           stripped = tree.slice %r{^/?(.*?)/?$}, 1 # strip both side /
           num_components = stripped.count('/')
           execute(
             :git, :archive, fetch(:branch), tree,
             "| tar -x --strip-components #{num_components} -f - -C ",
-            c.local_release_path
+            config.local_release_path
           )
         else
-          execute :git, :archive, fetch(:branch), '| tar -x -C', c.local_release_path
+          execute :git, :archive, fetch(:branch), '| tar -x -C', config.local_release_path
         end
       end
     end

data/lib/capistrano/net_storage/transport/base.rb

@@ -1,39 +1,27 @@
 module Capistrano
   module NetStorage
     module Transport
-      # Abstract class to transport archive from/to remote storage
-      # @abstract
+      # Abstract class to upload archive from local to storage and download from storage to remote
       class Base
-        # Check prerequisites for transport
-        # @abstract
         def check
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to check prerequisites for Transport"
         end
 
-        # Find uploaded archive to be deployed and set +:net_storage_uploaded_archive+ its address
-        # @abstract
-        def find_uploaded
-          raise NotImplementedError
+        def archive_exists?
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to test archive on storage corresponding to `fetch(:current_revision) + Config#archive_file_extension`"
         end
 
-        # Upload archive onto remote storage
-        # @abstract
         def upload
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to upload archive from `Capistrano::NetStorage.config.local_archive_path` to remote storage"
         end
 
-        # Download archive from remote storage to servers.
-        # Archive file should be placed at +config.archive_path+
-        # @abstract
         def download
-          raise NotImplementedError
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to download archive from remote storage to `Capistrano::NetStorage.config.archive_path`"
         end
 
-        # Clean up old archives on remote storage
-        # @abstract
-        # def cleanup
-        #   raise NotImplementedError
-        # end
+        def cleanup
+          raise NotImplementedError, "Implement `#{self.class}#{__method__}` to clean archives on remote storage to `Capistrano::NetStorage.config.keep_remote_archives`"
+        end
       end
     end
   end

data/lib/capistrano/net_storage/version.rb

@@ -1,5 +1,5 @@
 module Capistrano
   module NetStorage
-    VERSION = '0.4.0'
+    VERSION = '1.0.0'
   end
 end

data/lib/capistrano/net_storage.rb

@@ -1,2 +1,35 @@
-require 'capistrano/net_storage/all'
-load File.expand_path('../tasks/net_storage.rake', __FILE__)
+module Capistrano
+  module NetStorage
+    class << self
+      attr_reader :config
+
+      def setup!(config:)
+        @config = config
+      end
+
+      def transport
+        config.transport_class.new
+      end
+
+      def archiver
+        config.archiver_class.new
+      end
+
+      def scm
+        config.scm_class.new
+      end
+
+      def cleaner
+        config.cleaner_class.new
+      end
+
+      def bundler
+        config.bundler_class.new
+      end
+
+      def config_uploader
+        config.config_uploader_class.new
+      end
+    end
+  end
+end