luban 0.8.10 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93138388605dfd36238452d9c61eda0ba31c838f
-  data.tar.gz: bfe1d55d44ef71c128c855115aacd1a679b998a4
+  metadata.gz: b1c840fa119a9368e8e98cae2dfc701d2f31ead8
+  data.tar.gz: 10bc13a1c416506baae0a5aff9e88a17a61ee604
 SHA512:
-  metadata.gz: 93051b080c0bfca59e1f44fed83e752be9271b5849feb63927f50c94bbeef072165c1064ba779101c7f695e636406a419616b2b289ebe2624a0db342ed7bba17
-  data.tar.gz: 0e0aa51ed17722f15313f15490e739079efd8180c0e59883847e75cd2ebb4336cce0927b9d637741cee97f1497f5ff6cde50a5e7719f4cca12540cf5c0eda640
+  metadata.gz: 95ce838aecc0b93916935fc3469366826403599abcef453b8cada2b83063329eca02b7d0b25b0cbeb3e339a2cc5eb016f0f9e50c53947e64077ebfaf3d535718
+  data.tar.gz: af1aae4d01bb04f54097738a7a23c8b184842895e1e67be8705b95e15e523cbc9f1ae49ddf80dab129a1bad6691e5f5d0a3feb138258a2ac49161548b5b59ff9

data/CHANGELOG.md

@@ -1,5 +1,33 @@
 # Change log
 
+## Version 0.9.0 (Oct 12, 2016)
+
+New features:
+  * Added package for Logrotate
+    * Added dependent package for Popt
+  * Added support for log archival which is done thru Uber
+  * Added two convenient methods, #touch and #truncate, in module Utils
+
+Minor enhancements:
+  * Set keep_releases differently for app and profile publication
+  * Made application always deployable
+    * Application has/has no cronjobs to update which implies deployable
+  * Deprecated symlinks for logrotate configuration files
+    * As a result, directory "etc" is removed as well since it is no more necessary
+  * No more place holder cronjob entries if there are no more cronjobs defined
+  * Refactored profile templates handling in service configurator
+  * Cleaned up profile files that have been excluded
+  * Made luban_roles as the default roles when setting up cronjobs
+  * Added back support for build environment variables like LDFLAGS/CFLAGS
+  * Upgraded OpenSSL for Ruby to version 1.0.2j
+  * Upgraded Bundler for Ruby to version 1.13.3
+  * Added Rubygems dependency for Ruby (version >= 1.9.3)
+    * As a side effect, package option #rubygems for Ruby can be set for all Ruby versions
+  * Minor code cleanup
+
+Bug fixes:
+  * Added an extra validation for URL existence test
+
 ## Version 0.8.10 (Sept 29, 2016)
 
 Minor enhancements:

data/lib/luban/deployment/cli/application/base.rb

@@ -56,7 +56,7 @@ module Luban
       def has_services?; !services.empty?; end
 
       def provisionable?;  has_packages?; end
-      def deployable?;   has_source? or has_profile? end
+      def deployable?; true; end
       def controllable?; has_source? or has_services?; end
 
       def use_package?(package_name, package_version, servers: [])

data/lib/luban/deployment/cli/application/constructor.rb

@@ -20,13 +20,11 @@ module Luban
         end
 
         def destroy_project
-          rm(etc_path.join('*', "#{stage}.#{project}.*"))
           rmdir(project_path)
           update_result "The project environment is destroyed."
         end
 
         def destroy_app
-          rm(etc_path.join('*', "#{stage}.#{project}.#{application}.*"))
           rmdir(app_path)
           update_result "The application environment is destroyed."
         end
@@ -39,7 +37,7 @@ module Luban
         protected
 
         def bootstrap
-          assure_dirs(logrotate_path, downloads_path,
+          assure_dirs(downloads_path, archived_logs_path, 
                       tmp_path, app_bin_path, app_tmp_path, 
                       releases_path, shared_path)
         end

data/lib/luban/deployment/cli/application/crontab.rb

@@ -55,7 +55,11 @@ module Luban
             return
           end
           if updated?
-            update_result "Skipped! ALREADY updated crontab.", status: :skipped
+            if has_cronjobs?
+              update_result "Skipped! ALREADY updated crontab.", status: :skipped
+            else
+              update_result "Skipped! No crontab for #{user}.", status: :skipped
+            end
             return
           end
 
@@ -86,11 +90,19 @@ module Luban
 
         def deploy_cronjobs!
           rm(crontab_file_path) if force?
-          upload_by_template(file_to_upload: crontab_file_path,
-                             template_file:  crontab_template_file,
-                             header_file: crontab_header_template_file,
-                             footer_file: crontab_footer_template_file,
-                             auto_revision: true)
+          if has_cronjobs?
+            upload_by_template(file_to_upload: crontab_file_path,
+                               template_file:  crontab_template_file,
+                               header_file: crontab_header_template_file,
+                               footer_file: crontab_footer_template_file,
+                               auto_revision: true)
+          else
+            if file?(crontab_file_path, "-s")
+              truncate(crontab_file_path)
+            else
+              touch(crontab_file_path) unless file?(crontab_file_path)
+            end
+          end
         end
 
         def crontab_entry(command:, schedule:, output: "", type: :shell, disabled: false)
@@ -101,6 +113,7 @@ module Luban
           unless respond_to?(command_composer)
             abort "Aborted! Unknown cronjob type: #{type.inspect}"
           end
+          command = instance_exec(&command) if command.respond_to?(:call)
           command = send(command_composer, command, output: output)
           entry = "#{schedule} #{command}"
           disabled ? "# DISABLED - #{entry}" : entry
@@ -112,15 +125,21 @@ module Luban
           found = false
           crontab = crontab.split("\n").inject([]) do |lines, line|
             if found || line == crontab_open
-              lines << new_crontab unless (found = line != crontab_close)
+              unless (found = line != crontab_close)
+                lines << new_crontab unless new_crontab.empty?
+              end
             else
               lines << line
             end
             lines
           end
-          crontab << new_crontab unless crontab.include?(new_crontab)
-          upload!(StringIO.new(crontab.join("\n")), tmp_crontab_file_path)
-          test(:crontab, tmp_crontab_file_path, "2>&1")
+          crontab << new_crontab unless new_crontab.empty? or crontab.include?(new_crontab)
+          if crontab.empty?
+            test(:crontab, "-r", "2>&1")
+          else
+            upload!(StringIO.new(crontab.join("\n")), tmp_crontab_file_path)
+            test(:crontab, tmp_crontab_file_path, "2>&1")
+          end
         ensure
           rm(tmp_crontab_file_path)
         end

data/lib/luban/deployment/cli/application/publisher.rb

@@ -92,7 +92,7 @@ module Luban
 
         def publish!
           rollout_release
-          cleanup_releases
+          send("cleanup_#{release_type}_releases")
         end
 
         def rollout_release
@@ -100,7 +100,7 @@ module Luban
           upload!(release_package_path.to_s, upload_to.to_s)
           if md5_matched?(upload_to, release_md5) and
              test(:tar, "-xzf #{upload_to} -C #{releases_path}")
-            execute(:touch, release_path)
+            touch(release_path)
             create_symlinks
             update_releases_log
           else
@@ -119,7 +119,6 @@ module Luban
 
         def create_profile_symlinks
           create_release_symlink(shared_path)
-          create_etc_symlinks
         end
 
         def create_app_symlinks
@@ -135,23 +134,13 @@ module Luban
 
         def create_symlinks_for_linked_dirs
           create_linked_dirs(linked_dirs, from: shared_path, to: release_path)
+          assure_symlink(archived_logs_path, log_path.join(archived_logs_path.basename))
         end
 
         def create_symlinks_for_linked_files
           create_linked_files(linked_files, from: profile_path, to: release_path.join('config'))
         end
 
-        def create_etc_symlinks
-          create_logrotate_symlinks
-        end
-
-        def create_logrotate_symlinks
-          logrotate_files.each do |path|
-            target_file = "#{stage}.#{project}.#{application}.#{path.basename}"
-            assure_symlink(path, logrotate_path.join(target_file))
-          end
-        end
-
         def update_releases_log
           execute %{echo "[$(date -u)][#{user}] #{release_log_message}" >> #{releases_log_path}}
         end
@@ -160,13 +149,16 @@ module Luban
           "#{release_name} in #{stage} #{project} is published successfully."
         end
 
-        def cleanup_releases(keep_releases = 5)
+        def cleanup_releases(keep_releases = 1)
           files = capture(:ls, '-xtd', releases_path.join("#{release_version}-*")).split(" ")
           if files.count > keep_releases
             files.last(files.count - keep_releases).each { |f| rmdir(f) }
           end
         end
 
+        def cleanup_app_releases; cleanup_releases(5); end
+        def cleanup_profile_releases; cleanup_releases; end
+
         def bundle_gems
           assure_dirs(bundle_config_path, gems_cache_path, bundle_path)
           sync_gems_cache

data/lib/luban/deployment/cli/application/worker.rb

@@ -42,6 +42,15 @@ module Luban
         def bundle_command(cmd, **opts)
           shell_command("#{bundle_executable} exec #{cmd}", **opts)
         end
+
+        def app_archives_path
+          @app_archives_path ||= 
+            archives_path.join("#{stage}.#{project}", application, hostname)
+        end
+
+        def archived_logs_path
+          @archived_logs_path ||= app_archives_path.join('archived_logs')
+        end
       end
     end
   end

data/lib/luban/deployment/cli/command.rb

@@ -203,7 +203,7 @@ module Luban
 
           def has_cronjobs?; !cronjobs.empty?; end
 
-          def cronjob(roles: nil, hosts: nil, **job)
+          def cronjob(roles: luban_roles, hosts: nil, **job)
             validate_cronjob(job)
             roles = Array(roles)
             hosts = Array(hosts)

data/lib/luban/deployment/cli/package/installer/core.rb

@@ -46,8 +46,7 @@ module Luban
         def configure_build_env_variables          
           @build_env_vars = { 
             ldflags: [ENV['LDFLAGS'].to_s],
-            cflags: [ENV['CFLAGS'].to_s],
-            cppflags: [ENV['CPPFLAGS'].to_s]
+            cflags: [ENV['CFLAGS'].to_s]
           }
           if child?
             configure_parent_build_env_variables
@@ -58,7 +57,6 @@ module Luban
         def configure_parent_build_env_variables
           parent.build_env_vars[:ldflags] << "-L#{lib_path}"
           parent.build_env_vars[:cflags] << "-I#{include_path}"
-          parent.build_env_vars[:cppflags] << "-I#{include_path}"
         end
 
         def configure_parent_build_options
@@ -67,6 +65,13 @@ module Luban
           end
         end
 
+        def compose_build_env_variables
+          build_env_vars.inject({}) do |vars, (k, v)|
+            vars[k] = "#{v.join(' ').strip}" unless v.all?(&:empty?)
+            vars
+          end
+        end
+
         def configure_build_options
           @configure_opts = default_configure_opts
           @configure_opts |= task.opts.configure_opts unless task.opts.configure_opts.nil?

data/lib/luban/deployment/cli/package/installer/install.rb

@@ -302,7 +302,7 @@ module Luban
         end
 
         def bootstrap_install
-          assure_dirs(etc_path, tmp_path, app_bin_path, 
+          assure_dirs(tmp_path, app_bin_path, 
                       package_tmp_path, install_path, install_log_path)
         end
 

data/lib/luban/deployment/cli/service/configurator.rb

@@ -18,15 +18,23 @@ module Luban
               config_finder[:application].stage_profile_templates_path.join(profile_name)
           end
 
-          def profile_templates(format: "erb")
-            return @profile_templates unless @profile_templates.nil?
-            @profile_templates = []
+          def available_profile_templates(format: "erb")
+            return @available_profile_templates unless @available_profile_templates.nil?
+            templates = []
             [profile_templates_path, stage_profile_templates_path].each do |path|
-              Dir.chdir(path) { @profile_templates |= Dir["**/*.#{format}"] } if path.directory?
-            end
-            @profile_templates.tap do |templates| 
-              templates.reject! { |t| exclude_template?(t) }
+              Dir.chdir(path) { templates |= Dir["**/*.#{format}"] } if path.directory?
             end
+            @available_profile_templates = templates
+          end
+
+          def profile_templates(format: "erb")
+            @profile_templates ||= 
+              available_profile_templates(format: format).reject { |t| exclude_template?(t) }
+          end
+
+          def excluded_profile_templates(format: "erb")
+            @excluded_profile_templates ||= 
+              available_profile_templates(format: format).select { |t| exclude_template?(t) }
           end
 
           def exclude_template?(template); false; end
@@ -43,7 +51,7 @@ module Luban
           def update_profile
             assure_dirs(stage_profile_path)
             render_profile
-            update_logrotate_files
+            cleanup_profile
           end
 
           protected
@@ -79,9 +87,10 @@ module Luban
                                auto_revision: true)
           end
 
-          def update_logrotate_files
-            if file?(stage_profile_path.join(logrotate_file_name))
-              logrotate_files.push(logrotate_file_path)
+          def cleanup_profile
+            excluded_profile_templates.each do |template_file|
+              profile_file = stage_profile_path.join(template_file).sub_ext('')
+              rm(profile_file) if file?(profile_file)
             end
           end
         end

data/lib/luban/deployment/cli/service/worker.rb

@@ -73,14 +73,6 @@ module Luban
           def control_file_name
             @control_file_name ||= "#{service_name}.conf"
           end
-
-          def logrotate_file_path
-            @logrotate_file_path ||= profile_path.join(logrotate_file_name)
-          end
-
-          def logrotate_file_name
-            @logrotate_file_name ||= "#{service_name}.logrotate"
-          end
         end
 
         include Base

data/lib/luban/deployment/helpers/linked_paths.rb

@@ -23,7 +23,7 @@ module Luban
             assure_dirs(target_path.dirname)
             rmdir(target_path) if directory?(target_path)
             source_path = from.join(path)
-            assure_symlink(source_path, target_path) #if directory?(source_path)
+            assure_symlink(source_path, target_path) 
           end
         end
 
@@ -33,7 +33,7 @@ module Luban
             assure_dirs(target_path.dirname)
             rm(target_path) if file?(target_path)
             source_path = from.join(path)
-            assure_symlink(source_path, target_path) #if file?(source_path)
+            assure_symlink(source_path, target_path)
           end
         end
       end

data/lib/luban/deployment/helpers/utils.rb

@@ -52,6 +52,14 @@ module Luban
           execute(:mkdir, '-p', *opts, path)
         end
 
+        def truncate(path)
+          execute(:cat, "/dev/null", ">", path)
+        end
+
+        def touch(path)
+          execute(:touch, path)
+        end
+
         def rm(*opts, path)
           execute(:rm, '-f', *opts, path)
         end
@@ -106,13 +114,10 @@ module Luban
 
         def url_exists?(url)
           # Sent HEAD request to avoid downloading the file contents
-          test("curl -s -L -I -o /dev/null -f #{url}")
-
-          # Other effective ways to check url existence with curl
-
+          test("curl -s -L -I -o /dev/null -f #{url}") or
           # In case HEAD request is refused, 
           # only the first byte of the file is requested
-          # test("curl -s -L -o /dev/null -f -r 0-0 #{url}")
+          test("curl -s -L -o /dev/null -f -r 0-0 #{url}")
 
           # Alternatively, http code (200) can be validated
           # capture("curl -s -L -I -o /dev/null -w '%{http_code}' #{url}") == '200'

data/lib/luban/deployment/packages/logrotate.rb

@@ -0,0 +1,43 @@
+module Luban
+  module Deployment
+    module Packages
+      class Logrotate < Luban::Deployment::Package::Base
+        apply_to :all do
+          before_install do
+            depend_on 'popt', version: '1.16'
+          end
+        end
+
+        class Installer < Luban::Deployment::Package::Installer
+          define_executable 'logrotate'
+
+          def source_repo
+            @source_repo ||= "https://github.com"
+          end
+
+          def source_url_root
+            @source_url_root ||= "logrotate/logrotate/releases/download/#{package_major_version}"
+          end
+
+          def installed?
+            return false unless file?(logrotate_executable)
+            pattern = "logrotate #{package_major_version}"
+            match?("#{logrotate_executable} --version 2>&1", pattern)
+          end
+
+          def bin_path
+            @bin_path ||= install_path.join('sbin')
+          end
+
+          protected
+
+          def configure_package!
+            with compose_build_env_variables do
+              test("./autogen.sh", ">> #{install_log_file_path} 2>&1") and super
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/luban/deployment/packages/popt.rb

@@ -0,0 +1,38 @@
+module Luban
+  module Deployment
+    module Packages
+      class Popt < Luban::Deployment::Package::Base
+        class Installer < Luban::Deployment::Package::Installer
+          def header_file
+            @header_file ||= include_path.join('popt.h')
+          end
+
+          def shared_obj_file
+            @shared_obj_file ||= lib_path.join("libpopt.#{lib_extension}")
+          end
+
+          def source_repo
+            @source_repo ||= "http://rpm5.org"
+          end
+
+          def source_url_root
+            @source_url_root ||= "files/popt"
+          end
+
+          def installed?
+            file?(header_file) and file?(shared_obj_file)
+          end
+
+          protected
+
+          def configure_build_options
+            super
+            @configure_opts.unshift("--disable-static")
+          end
+          
+          def update_binstubs!; end
+        end
+      end
+    end
+  end
+end

data/lib/luban/deployment/packages/ruby.rb

@@ -46,14 +46,15 @@ module Luban
 
         apply_to '>= 1.9.3' do
           before_install do
-            depend_on 'openssl', version: '1.0.2h'
+            depend_on 'openssl', version: '1.0.2j'
+            depend_on 'rubygems', version: '2.6.7'
             #depend_on 'yaml', version: '0.1.6'
           end
         end
 
         apply_to :all do
           after_install do
-            depend_on 'bundler', version: '1.12.3'
+            depend_on 'bundler', version: '1.13.3'
           end
         end
         
@@ -72,7 +73,7 @@ module Luban
           provision_tasks[:install].switch :install_doc, "Install Ruby document"
           provision_tasks[:install].switch :install_tcl, "Install with Tcl"
           provision_tasks[:install].switch :install_tk, "Install with Tk"
-          provision_tasks[:install].option :rubygems, "Rubygems version (effective for v1.9.2 or below)"
+          provision_tasks[:install].option :rubygems, "Rubygems version"
           provision_tasks[:install].option :bundler, "Bundler version"
           provision_tasks[:install].option :openssl, "OpenSSL version (effective for v1.9.3 or above)"
         end

data/lib/luban/deployment/packages/rubygems.rb

@@ -33,7 +33,6 @@ module Luban
           end
 
           def source_repo
-            #@source_repo ||= "http://production.cf.rubygems.org"
             @source_repo ||= "https://rubygems.org"
           end
 

data/lib/luban/deployment/parameters.rb

@@ -119,7 +119,6 @@ module Luban
 
         parameter :application
         parameter :scm_role
-        parameter :logrotate_files
 
         def env_name
           @env_name ||= "#{stage}.#{project}/#{application}"
@@ -137,8 +136,6 @@ module Luban
 
         def set_default_application_parameters
           set_default :scm_role, :scm
-          set_default :logrotate_files, []
-
           setup_default_application_config_finder
         end
 

data/lib/luban/deployment/templates/crontab.erb

@@ -1,6 +1,3 @@
-<% unless has_cronjobs? -%>
-# !!! No cronjobs for <%= user %> !!!
-<% end -%>
 <% cronjobs.each do |cronjob| -%>
 <%= "#{crontab_entry(**cronjob)}" %>
 <% end -%>

data/lib/luban/deployment/version.rb

@@ -1,5 +1,5 @@
 module Luban
   module Deployment
-    VERSION = "0.8.10"
+    VERSION = "0.9.0"
   end
 end

data/lib/luban/deployment/worker/paths.rb

@@ -21,13 +21,10 @@ module Luban
             @env_path ||= luban_root_path.join('env')
           end
 
-          def etc_path
-            @etc_path ||= luban_root_path.join('etc')
-          end
-
-          def logrotate_path
-            @logrotate_path = etc_path.join('logrotate')
+          def archives_path
+            @archives_path ||= luban_root_path.join('archives')
           end
+          alias_method :local_archives_path, :archives_path
 
           def tmp_path
             @tmp_path ||= luban_root_path.join('tmp')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: luban
 version: !ruby/object:Gem::Version
-  version: 0.8.10
+  version: 0.9.0
 platform: ruby
 authors:
 - Rubyist Lei
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-29 00:00:00.000000000 Z
+date: 2016-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: luban-cli
@@ -143,7 +143,9 @@ files:
 - lib/luban/deployment/helpers/utils.rb
 - lib/luban/deployment/packages/bundler.rb
 - lib/luban/deployment/packages/git.rb
+- lib/luban/deployment/packages/logrotate.rb
 - lib/luban/deployment/packages/openssl.rb
+- lib/luban/deployment/packages/popt.rb
 - lib/luban/deployment/packages/ruby.rb
 - lib/luban/deployment/packages/rubygems.rb
 - lib/luban/deployment/packages/yaml.rb
@@ -200,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.7
 signing_key: 
 specification_version: 4
 summary: Ruby framework for server automation and application deployment