luban 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f3bb8577686b082a631040457727ce47b0999543
+  data.tar.gz: 657f3f393403953ea5606b288f997f212beb4ab3
+SHA512:
+  metadata.gz: 7344a5bdf345d666bed583166e90ed40b773fb2f7bdb224102b1d2eefc2de24a270d4b5ae23dbe1eb6fc4a36538ec100d335730134efe25c507474e39b659ff2
+  data.tar.gz: 960aa2296bb31efdae8025a9180188867b94603871977ca307c3b6ac35540d5e99f993115c0e54b38a34849af9e68595d153e399ba25b81461efa3605db0fe8f

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in luban.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Chi Man Lei
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# Luban
+
+Luban is a framework to manage server automation and application deployment.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'luban'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install luban
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment. Run `bundle exec luban` to use the code located in this directory, ignoring other installed copies of this gem.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/luban/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "luban"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/luban

@@ -0,0 +1,3 @@
+require "luban"
+
+Luban::Deployment::Runner.new.run

data/lib/luban/deployment/cli/application/authenticator.rb

@@ -0,0 +1,106 @@
+module Luban
+  module Deployment
+    class Application
+      class Authenticator < Luban::Deployment::Worker::Base
+        def private_key_file_name
+          @private_key_file_name ||= "id_#{authen_key_type}"
+        end
+
+        def authen_keys_path
+          @authen_keys_path ||= Pathname.new(user_home).join('.ssh')
+        end
+
+        def private_key_file_path
+          @private_key_file ||= authen_keys_path.join(private_key_file_name)
+        end
+
+        def public_key_file_path
+          @public_key_file_path ||= authen_keys_path.join("#{private_key_file_name}.pub")
+        end
+
+        def authorized_keys_file_path
+          @authorized_keys_file_path ||= authen_keys_path.join('authorized_keys')
+        end
+
+        def public_key
+          @public_key ||= get_public_key
+        end
+
+        def keyget_command
+          @keyget_command ||= "cat #{public_key_file_path} 2>&1"
+        end
+
+        def keygen_command
+          @keygen_command ||= "ssh-keygen -t #{authen_key_type} -f #{private_key_file_path} -N '' 2>&1"
+        end
+
+        def get_public_key
+          generate_key_pairs
+          capture(keyget_command)
+        end
+
+        def generate_key_pairs
+          execute(keygen_command) unless key_pairs_generated?
+        end
+
+        def key_pairs_generated?
+          file?(private_key_file_path) and file?(public_key_file_path)
+        end
+
+        def app; task.opts.app; end
+
+        def promptless_authen
+          if promptless_authen_enabled?
+            update_result "Skipped! Promptless authentication has been enabled ALREADY.",
+                          status: :skipped, public_key: public_key
+          else
+            setup_password_authen
+            generate_key_pairs
+            add_authorized_keys
+            update_result "Promptless authentication is enabled.", public_key: public_key
+          end
+        end
+
+        def promptless_authen_enabled?
+          origin_auth_methods = host.ssh_options[:auth_methods]
+          host.ssh_options[:auth_methods] = %w(publickey)
+          capture('echo ok') == 'ok'
+        rescue Net::SSH::AuthenticationFailed
+          false
+        ensure
+          if origin_auth_methods.nil?
+            host.ssh_options.delete(:auth_methods)
+          else
+            host.ssh_options[:auth_methods] = origin_auth_methods
+          end
+        end
+
+        protected
+
+        def setup_password_authen
+          host.user, host.password = user, nil if host.user.nil?
+          host.password = app.password_for(host.user) if host.password.nil?
+          host.ssh_options[:auth_methods] = %w(keyboard-interactive)
+        end
+
+        def add_authorized_keys
+          public_keys = task.opts.public_keys || []
+          public_keys.uniq!
+          if file?(authorized_keys_file_path)
+            public_keys.each { |k| add_authorized_key(k) unless key_authorized?(k) }
+          else
+            public_keys.each { |k| add_authorized_key(k) }
+          end
+        end
+
+        def add_authorized_key(key)
+          execute("umask 077; echo #{key} >> #{authorized_keys_file_path} 2>&1")
+        end
+
+        def key_authorized?(key)
+          test("grep -v \"^#\" #{authorized_keys_file_path} | grep -Fxq \"#{key}\"")
+        end
+      end
+    end
+  end
+end

data/lib/luban/deployment/cli/application/base.rb

@@ -0,0 +1,179 @@
+module Luban
+  module Deployment
+    class Application < Luban::Deployment::Command
+      include Luban::Deployment::Parameters::Project
+      include Luban::Deployment::Parameters::Application
+      include Luban::Deployment::Command::Tasks::Install
+      include Luban::Deployment::Command::Tasks::Deploy
+      include Luban::Deployment::Command::Tasks::Control
+
+      attr_reader :packages
+      attr_reader :services
+
+      def has_source?;   !source.empty?; end
+      def has_profile?;  !profile.empty?; end
+      def has_packages?; !packages.empty?; end
+      def has_services?; !services.empty?; end
+
+      def installable?;  has_source? or has_packages?; end
+      def deployable?;   has_source?  or has_profile? or has_services?; end
+      def controllable?; has_source? or has_profile? or has_services?; end
+
+      def use_package?(package_name, package_version, servers: [])
+        package_name = package_name.to_sym
+        packages.has_key?(package_name) and
+        packages[package_name].has_version?(package_version) and
+        packages[package_name].config.servers.any? { |s| servers.include?(s) }
+      end
+
+      def other_package_users_for(package_name, package_version, servers: [])
+        parent.package_users_for(package_name, package_version, 
+                                 exclude: [name], servers: servers)
+      end
+
+      def package(name, version:, **opts)
+        yield opts if block_given?
+        name = name.to_sym
+        pkg = if has_command?(name)
+                commands[name]
+              else
+                command(name, base: Luban::Deployment::Package::Base.package_class(name))
+              end
+        pkg.update_package_options(version, opts)
+        services[name] = pkg if pkg.is_a?(Luban::Deployment::Package::Service)
+        packages[name] = pkg
+      end
+      alias_method :require_package, :package
+
+      def source(from = nil, **opts)
+        from.nil? ? @source : (@source = opts.merge(name: 'app', from: from))
+      end
+
+      def profile(from = nil, **opts)
+        from.nil? ? @profile : (@profile = opts.merge(name: 'profile', from: from))
+      end
+
+      def password_for(user); parent.password_for(user); end
+
+      def promptless_authen(args:, opts:)
+        opts = opts.merge(app: self, public_keys: Array(public_key!(args: args, opts: opts)))
+        opts[:roles] << scm_role unless scm_role.nil?
+        promptless_authen!(args: args, opts: opts)
+      end
+
+      def build(args:, opts:)
+        show_app_environment
+        promptless_authen(args: args, opts: opts)
+        build_repositories(args: args, opts: opts)
+        build!(args: args, opts: opts)
+        install_all(args: args, opts: opts)
+      end
+
+      def destroy(args:, opts:)
+        uninstall_all(args: args, opts: opts)
+        destroy!(args: args, opts: opts)
+      end
+
+      %i(install_all uninstall_all).each do |action|
+        define_method(action) do |args:, opts:|
+          packages.each_value { |p| p.send(__method__, args: args, opts: opts) }
+        end
+      end
+
+      %i(cleanup binstubs show_current show_summary which whence).each do |action|
+        define_method(action) do |args:, opts:|
+          show_app_environment
+          packages.each_value { |p| p.send(__method__, args: args, opts: opts) }
+        end
+      end
+
+      alias_method :cleanup_packages, :cleanup
+      def cleanup(args:, opts:)
+        cleanup_packages(args: args, opts: opts)
+        cleanup!(args: args, opts: opts)
+      end
+
+      def destroy_project(args:, opts:)
+        show_app_environment
+        destroy!(args: args, opts: opts.merge(destroy_project: true))
+      end
+
+      def deploy(args:, opts:)
+        show_app_environment
+        deploy_releases(args: args, opts: opts)
+      end
+
+      protected
+
+      def set_parameters
+        super
+        copy_parameters_from_parent(:stage, :project)
+        @packages = {}
+        @services = {}
+        @source = {}
+        @profile = {}
+      end
+
+      def validate_parameters
+        super
+        validate_project_parameters
+        validate_application_parameters
+      end
+
+      def set_default_parameters
+        super
+        set_default_project_parameters
+        set_default :application, self.class.name.split(':').last.downcase
+        set_default_application_parameters
+        set_default_profile
+      end
+
+      def set_default_profile
+        profile_path = config_finder[:application].stage_config_path.join('profile')
+        profile(profile_path, scm: :rsync) if profile_path.directory?
+      end
+
+      def load_configuration
+        config_finder[:project].load_configuration
+        config_finder[:application].load_configuration
+      end
+
+      def setup_descriptions
+        desc "Manage application #{display_name}"
+        long_desc "Manage the deployment of application #{display_name} in #{parent.class.name}"
+      end
+
+      def show_app_environment
+        puts "#{display_name} in #{parent.class.name}"
+      end
+
+      %i(build destroy cleanup).each do |m|
+        define_task_method("#{m}!", task: m, worker: :builder)
+      end
+
+      def build_repositories(args:, opts:)
+        build_repository!(args: args, opts: opts.merge(repository: profile)) if has_profile?
+        build_repository!(args: args, opts: opts.merge(repository: source)) if has_source?
+      end
+
+      def deploy_releases(args:, opts:)
+        deploy_release(args: args, opts: opts.merge(repository: profile)) if has_profile?
+        deploy_release(args: args, opts: opts.merge(repository: source)) if has_source?
+      end
+
+      def deploy_release(args:, opts:)
+        package_release!(args: args, opts: opts)[:release].tap do |release|
+          unless release.nil?
+            publish_release!(args: args, opts: opts.merge(release: release))
+          end
+        end
+      end
+
+      define_task_method("promptless_authen!", task: :promptless_authen, worker: :authenticator)
+      define_task_method("public_key!", task: :public_key, worker: :authenticator, locally: true)
+      define_task_method("build_repository!", task: :build, worker: :repository, locally: true)
+      define_task_method("package_release!", task: :package, worker: :repository, locally: true)
+      define_task_method("publish_release!", task: :publish, worker: :publisher)
+    end
+  end
+end

data/lib/luban/deployment/cli/application/builder.rb

@@ -0,0 +1,67 @@
+module Luban
+  module Deployment
+    class Application
+      class Builder < Luban::Deployment::Worker::Remote
+        def envrc_template_file
+          @envrc_template_file ||= find_template_file("envrc.erb")
+        end
+
+        def unset_envrc_template_file
+          @unset_envrc_template_file ||= find_template_file("unset_envrc.erb")
+        end
+
+        def build
+          bootstrap
+          create_envrc_files
+        end
+
+        def destroy
+          task.opts.destroy_project ? destroy_project : destroy_app
+        end
+
+        def destroy_project
+          rm(etc_path.join('*', "#{stage}.#{project}.*"))
+          rmdir(project_path)
+          update_result "The project environment is destroyed."
+        end
+
+        def destroy_app
+          rm(etc_path.join('*', "#{stage}.#{project}.#{application}.*"))
+          rm(app_path)
+          update_result "The application environment is destroyed."
+        end
+
+        def cleanup
+          execute("find #{tmp_path}/* -type f|xargs rm -f")
+          update_result "Temporary files in app environment is cleaned up."
+        end
+
+        protected
+
+        def bootstrap
+          assure_dirs(etc_path, tmp_path, 
+                      app_bin_path, app_tmp_path,
+                      releases_path, shared_path)
+          assure_linked_dirs
+        end
+
+        def assure_linked_dirs
+          return if linked_dirs.empty?
+          linked_dirs.each do |dir|
+            linked_dir = shared_path.join(dir)
+            assure(:directory, linked_dir) { mkdir(linked_dir) }
+          end
+        end
+
+        def create_envrc_files
+          upload_by_template(file_to_upload: envrc_file,
+                             template_file:  envrc_template_file,
+                             auto_revision: true)
+          upload_by_template(file_to_upload: unset_envrc_file,
+                             template_file:  unset_envrc_template_file,
+                             auto_revision: true)
+        end
+      end
+    end
+  end
+end

data/lib/luban/deployment/cli/application/publisher.rb

@@ -0,0 +1,215 @@
+module Luban
+  module Deployment
+    class Application
+      class Publisher < Luban::Deployment::Worker::Remote
+        def release_name; task.opts.release[:name]; end
+        def release_tag; task.opts.release[:tag]; end
+        def release_package_path; task.opts.release[:path]; end
+        def release_md5; task.opts.release[:md5]; end
+        def bundled_gems; task.opts.release[:bundled_gems]; end
+        def locked_gemfile; bundled_gems[:locked_gemfile]; end
+        def gems_source; bundled_gems[:gems_cache]; end
+        def gems; bundled_gems[:gems]; end
+
+        def publish_app?; release_name == 'app'; end
+        def publish_profile?; release_name == 'profile'; end
+
+        def display_name
+          @display_name ||= "#{application} #{release_name} (release: #{release_tag})"
+        end
+
+        def releases_path
+          @releases_path ||= super.join(release_name)
+        end
+
+        def release_path
+          @release_path ||= releases_path.join(release_tag)
+        end
+
+        def releases_log_path
+          @releases_log_path ||= app_path.join('releases.log')
+        end
+
+        def gemfile
+          @gemfile ||= release_path.join('Gemfile')
+        end
+
+        def bundle_cmd
+          @bundle_cmd ||= app_bin_path.join('bundle')
+        end
+
+        def bundle_config_path
+          @bundle_config_path ||= shared_path.join('.bundle')
+        end
+
+        def bundle_path
+          @bundle_path ||= shared_path.join('vendor').join('bundle')
+        end
+
+        def gems_cache_path
+          @gems_cache_path ||= shared_path.join('vendor').join('cache')
+        end
+
+        def bundle_without
+          @bundle_without ||= %w(development test)
+        end
+
+        def bundle_flags
+          @bundle_flags ||= %w(--deployment --quiet)
+        end
+
+        def bundle_linked_dirs
+          @bundle_linked_dirs ||= %w(.bundle vendor/cache vendor/bundle)
+        end
+
+        def published?
+          get_releases.include?(release_tag)
+        end
+
+        def publish
+          assure_dirs(releases_path)
+          if published?
+            if force?
+              publish!
+            else
+              update_result "Skipped! #{display_name} has been published ALREADY.", status: :skipped
+              return
+            end
+          else
+            publish!
+          end
+
+          if published?
+            update_result "Successfully published #{display_name}."
+          else
+            update_result "Failed to publish #{display_name}", status: :failed, level: :error
+          end
+        end
+
+        def after_publish
+          create_symlinks
+          bundle_gems
+        end
+
+        protected
+
+        def get_releases
+          capture(:ls, '-xt', releases_path).split
+        end
+
+        def publish!
+          rollout_release
+          cleanup_releases
+        end
+
+        def rollout_release
+          upload_to = app_tmp_path.join(release_package_path.basename)
+          upload!(release_package_path.to_s, upload_to.to_s)
+          if md5_matched?(upload_to, release_md5) and
+             test(:tar, "-xzf #{upload_to} -C #{releases_path}")
+            create_symlinks
+            update_releases_log
+          else
+            rm(release_path)
+          end
+        ensure
+          rm(upload_to)
+        end
+
+        def create_symlinks
+          send("create_#{release_name}_symlinks")
+        end
+
+        def create_profile_symlinks
+          create_release_symlink(shared_path)
+          create_shared_symlinks_for(:directory, linked_dirs)
+          create_shared_symlinks_for(:directory, bundle_linked_dirs) if file?(gemfile)
+        end
+
+        def create_app_symlinks
+          create_release_symlink(app_path)
+          create_shared_symlinks_for(:directory, linked_dirs | %w(profile))
+          create_shared_symlinks_for(:directory, bundle_linked_dirs) if file?(gemfile)
+          create_shared_symlinks_for(:file, linked_files)
+        end
+
+        def create_release_symlink(target_dir)
+          assure_symlink(release_path, target_dir.join(release_name))
+        end
+
+        def create_shared_symlinks_for(type, linked_paths)
+          linked_paths.each do |path|
+            target_path = release_path.join(path)
+            assure_dirs(target_path.dirname)
+            source_path = shared_path.join(path)
+            assure_symlink(source_path, target_path)
+          end
+        end
+
+        def update_releases_log
+          execute %{echo "[$(date -u)][#{user}] #{release_log_message}" >> #{releases_log_path}}
+        end
+
+        def release_log_message
+          "Release #{display_name} in #{stage} #{project} is published successfully."
+        end
+
+        def cleanup_releases
+          releases = get_releases
+          if releases.count > keep_releases
+            releases_to_keep = releases.first(keep_releases)
+            unless releases_to_keep.include?(release_tag)
+              releases_to_keep[-1] = release_tag
+            end
+            releases_to_remove = releases - releases_to_keep
+            releases_to_remove.each do |release| 
+              rm(releases_path.join(release))
+            end
+            info "Removed #{releases_to_remove.count} old releases."
+          else
+            info "No old releases to remove (keeping most recent #{keep_releases} releases)."
+          end
+        end
+
+        def bundle_gems
+          assure_dirs(bundle_config_path, gems_cache_path, bundle_path)
+          sync_gems_cache
+          sync_locked_gemfile
+          install_gems_from_cache
+        end
+
+        def sync_gems_cache
+          capture(:ls, '-xt', gems_cache_path).split.each do |gem_name|
+            rm(gems_cache_path.join(gem_name)) unless gems.has_key?(gem_name)
+          end
+          gems.each_pair do |gem_name, md5|
+            gem_path = gems_cache_path.join(gem_name)
+            unless md5_matched?(gem_path, md5)
+              upload!(gems_source.join(gem_name).to_s, gem_path.to_s)
+            end
+          end
+        end
+
+        def sync_locked_gemfile
+          gemfile_lock_path = release_path.join(locked_gemfile[:path].basename)
+          unless md5_matched?(gemfile_lock_path, locked_gemfile[:md5])
+            upload!(locked_gemfile[:path].to_s, gemfile_lock_path.to_s)
+          end
+        end
+
+        def install_gems_from_cache
+          within(release_path) do
+            options = []
+            options << "--gemfile #{gemfile}"
+            options << "--path #{bundle_path}"
+            unless test(bundle_cmd, :check, *options)
+              options << "--without #{bundle_without.join(' ')}"
+              options << bundle_flags.join(' ')
+              execute(bundle_cmd, :install, *options)
+            end
+          end
+        end
+      end
+    end
+  end
+end