itamae-mitsurin 0.50 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 24e20da5d3a61cb0db137e79d80c5ef5e47951f2
-  data.tar.gz: 50ade042e38c73694307396f626b2fe37135262b
+SHA256:
+  metadata.gz: 820a66a01e1d3908d4c24206171897a2931cc7063a141fad22959206ce507909
+  data.tar.gz: 374e9edf28cf7a2024383045cea7cc016cb00c43e4e5bd0513d3fe8a510165b8
 SHA512:
-  metadata.gz: d6718987fe3f358415af4711389f3107e7e38f64f72aa875042136290aafca9ff1248d70bc60888d190c092a775db9125d7f6d9dcbd3de8a5fb8f8dd97efc596
-  data.tar.gz: f5e5dfdb90173448bda7771e761e1cb7b3c582c3877bd7d987406481a256ed4ea54391732518238b70a608e5dc1ed575d31331216750189faad1bec9591438f3
+  metadata.gz: 90925c472f20db377cc5b1831f2495ea1499c6a4362c43cb8b39dde4271ae3db1eb13eac9baf088e0116a4d52fce952eefe907d21447e3c556a84d3ed97f054a
+  data.tar.gz: 4f853e11a25179818bd250884b6f0c1ce275c42b5a0a0f53afe2821e4348048911efc3d16aa34e737eba502d8c8461961b9a26703ef2d323728fb0b212c09144

data/lib/itamae-mitsurin/cli.rb

@@ -12,9 +12,7 @@ module ItamaeMitsurin
       super
 
       ItamaeMitsurin.logger.level = ::Logger.const_get(options[:log_level].upcase)
-      ItamaeMitsurin.file_logger.level = ::Logger.const_get(options[:log_level].upcase)
       ItamaeMitsurin.logger.formatter.colored = options[:color]
-      ItamaeMitsurin.file_logger.formatter.colored = options[:color]
     end
 
     def self.define_exec_options

data/lib/itamae-mitsurin/logger.rb

@@ -118,26 +118,88 @@ module ItamaeMitsurin
         ANSI.public_send(color_code) { str }
       end
     end
-  end
 
- # @logger = ::Logger.new($stdout).tap do |l|
- #   l.formatter = ItamaeMitsurin::Logger::Formatter.new
- # end.extend(ItamaeMitsurin::Logger::Helper)
+    class FileFormatter < Formatter
+      def call(severity, datetime, progname, msg)
+        log = "%s : %s" % ["%5s" % severity, msg2str(msg)]
+        Time.now.strftime('%F %T %z').to_s + log + "\n"
+      end
 
-  @logger = ::Logger.new($stdout).tap do |l|
-    l.formatter = ItamaeMitsurin::Logger::Formatter.new
-  end.extend(ItamaeMitsurin::Logger::Helper)
+      # def colorize(str, severity)
+      #   Time.now.strftime('%F %T %z').to_s + str
+      # end
+    end
+
+    def self.broadcast(logger)
+      Module.new do
+        define_method(:add) do |*args, &block|
+          logger.add(*args, &block)
+          super(*args, &block)
+        end
+
+        define_method(:<<) do |x|
+          logger << x
+          super(x)
+        end
+
+        define_method(:close) do
+          logger.close
+          super()
+        end
+
+        define_method(:progname=) do |name|
+          logger.progname = name
+          super(name)
+        end
+
+        define_method(:formatter=) do |formatter|
+          logger.formatter = formatter
+          super(formatter)
+        end
+
+        define_method(:level=) do |level|
+          logger.level = level
+          super(level)
+        end
+
+        define_method(:local_level=) do |level|
+          logger.local_level = level if logger.respond_to?(:local_level=)
+          super(level) if respond_to?(:local_level=)
+        end
 
-  class ItamaeMitsurin::Logger::FileFormatter < ItamaeMitsurin::Logger::Formatter
-    def colorize(str, severity)
-      Time.now.strftime('%Y %m %d %H:%M:%S %z').to_s + str
+        define_method(:silence) do |level = Logger::ERROR, &block|
+          if logger.respond_to?(:silence)
+            logger.silence(level) do
+              if defined?(super)
+                super(level, &block)
+              else
+                block.call(self)
+              end
+            end
+          else
+            if defined?(super)
+              super(level, &block)
+            else
+              block.call(self)
+            end
+          end
+        end
+      end
     end
   end
 
-  @file_logger = ::Logger.new('logs/itamae.log', 5, 100 * 1024 * 1024).tap do |l|
-    l.formatter = ItamaeMitsurin::Logger::FileFormatter.new
+  @logger = ::Logger.new($stdout).tap do |l|
+    l.formatter = ItamaeMitsurin::Logger::Formatter.new
   end.extend(ItamaeMitsurin::Logger::Helper)
 
+  if Dir.exist?('logs')
+    @file_logger = ::Logger.new('logs/itamae.log', 'daily').tap do |l|
+      l.formatter = ItamaeMitsurin::Logger::FileFormatter.new
+    end.extend(ItamaeMitsurin::Logger::Helper)
+
+    @logger.extend ItamaeMitsurin::Logger.broadcast(@file_logger)
+  end
+
   class << self
     def logger
       @logger
@@ -146,13 +208,5 @@ module ItamaeMitsurin
     def logger=(l)
       @logger = l.extend(ItamaeMitsurin::Logger::Helper)
     end
-
-    def file_logger
-      @file_logger
-    end
-
-    def file_logger=(l)
-      @file_logger = l.extend(ItamaeMitsurin::Logger::Helper)
-    end
   end
 end

data/lib/itamae-mitsurin/mitsurin/base.rb

@@ -0,0 +1,109 @@
+require 'itamae-mitsurin'
+require 'json'
+require 'highline'
+require 'tmpdir'
+require 'logger'
+
+module ItamaeMitsurin
+  module Mitsurin
+    module Base
+      RoleLoadError = Class.new(StandardError)
+
+      class ::Hash
+        def deep_merge(other_hash, &block)
+          dup.deep_merge!(other_hash, &block)
+        end
+
+        def deep_merge!(other_hash, &block)
+          merge!(other_hash) do |key, this_val, other_val|
+            if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+              this_val.deep_merge(other_val, &block)
+            elsif block_given?
+              yield(key, this_val, other_val)
+            else
+              other_val
+            end
+          end
+        end
+      end
+
+      class << self
+        # def get_roles(node_file)
+        #   roles = []
+        #   JSON.parse(File.read(node_file))['run_list'].each do |role|
+        #     roles << role.gsub(/role\[(.+)\]/, '\1') if /role\[(.+)\]/.match?(role)
+        #   end
+        #
+        #   roles
+        # end
+
+        def get_role_recipes(role)
+          recipes = []
+          JSON.parse(File.read("roles/#{role}.json"))['run_list'].each do |recipe|
+            if /recipe\[(.+)::(.+)\]/.match?(recipe)
+              recipes << { recipe.gsub(/recipe\[(.+)::(.+)\]/, '\1') => recipe.gsub(/recipe\[(.+)::(.+)\]/, '\2') }
+            elsif /recipe\[(.+)\]/.match?(recipe)
+              recipes << { recipe.gsub(/recipe\[(.+)\]/, '\1') => 'default' }
+            end
+          end
+        rescue JSON::ParserError
+          raise RoleLoadError, "JSON Parser Faild. - roles/#{role}.json"
+        rescue Errno::ENOENT
+          raise RoleLoadError, "No such role file or directory - roles/#{role}.json"
+        else
+          recipes
+        end
+
+        def get_node_recipes(node_file)
+          recipes = []
+          JSON.parse(File.read(node_file))['run_list'].each do |recipe|
+            if /recipe\[(.+)::(.+)\]/.match?(recipe)
+              recipes << { recipe.gsub(/recipe\[(.+)::(.+)\]/, '\1') => recipe.gsub(/recipe\[(.+)::(.+)\]/, '\2') }
+            elsif /recipe\[(.+)\]/.match?(recipe)
+              recipes << { recipe.gsub(/recipe\[(.+)\]/, '\1') => 'default' }
+            elsif /role\[(.+)\]/.match?(recipe)
+              recipes << get_role_recipes(recipe.gsub(/role\[(.+)\]/, '\1'))
+            end
+          end
+        rescue JSON::ParserError
+          raise RoleLoadError, "JSON Parser Faild. - #{node_file}"
+        rescue Errno::ENOENT
+          raise RoleLoadError, "No such node file or directory - #{node_fie}"
+        else
+          recipes
+        end
+
+        def jq(*objs)
+          par = nil
+          objs.each {|obj| par = JSON.pretty_generate(obj, allow_nan: true, max_nesting: false) }
+          par
+        end
+
+        def write_tmp_nodes(filename)
+          ItamaeMitsurin.logger.info "Output attributes log file to: tmp-nodes/#{filename}.json"
+
+          File.open "tmp-nodes/#{filename}.json", 'w' do |f|
+            f.flock File::LOCK_EX
+            yield f
+            f.flock File::LOCK_UN
+          end
+        end
+
+        def write_tmp_json(filename)
+          path = Dir.mktmpdir('mitsurin-')
+          open("#{path}/#{filename}.json", 'w') do |f|
+            f.flock File::LOCK_EX
+            yield f
+            f.flock File::LOCK_UN
+          end
+
+          path
+        end
+
+        def handler_logger
+          default = {"handlers"=>[{"type"=>"json", "path"=>"itamae-log.json"}]}
+        end
+      end
+    end
+  end
+end

data/lib/itamae-mitsurin/mitsurin/base_task.rb

@@ -0,0 +1,147 @@
+require 'itamae-mitsurin/mitsurin/base'
+
+module ItamaeMitsurin
+  module Mitsurin
+    class BaseTask
+      extend Rake::DSL if defined? Rake::DSL
+      EnvironmentsSetError = Class.new(StandardError)
+      LoadRecipeError = Class.new(StandardError)
+      LoadAttributeError = Class.new(StandardError)
+
+      def load_node_attributes(node_file)
+        JSON.parse(File.read(node_file), symbolize_names: true)
+      rescue JSON::ParserError
+        raise LoadAttributeError, "JSON Parser Failed. - #{node_file}"
+      end
+
+      def load_run_list(node_file)
+        run_list = []
+        run_list << { '_base' => 'default' }
+        # Base.get_roles(node_file).each {|role| run_list << Base.get_role_recipes(role) }
+        Base.get_node_recipes(node_file).each {|recipe| run_list << recipe }
+        run_list.flatten
+      end
+
+      def load_environments(hash)
+        set = hash[:environments][:set]
+        raise EnvironmentsSetError, 'Environments Set is not specified in nodefile' if set.nil?
+        JSON.parse(File.read("environments/#{set}.json"), symbolize_names: true)
+      rescue JSON::ParserError
+        raise LoadAttributeError, "JSON Parser Failed. - environments/#{set}.json"
+      end
+
+      def load_recipe_attributes(run_list)
+        recipe_files = run_list.map do |recipe|
+          Dir.glob("site-cookbooks/**/#{recipe.keys.join}/attributes/#{recipe.values.join}.json")
+        end.flatten
+
+        recipe_files.map do |f|
+          begin
+            JSON.parse(File.read(f), symbolize_names: true)
+          rescue JSON::ParserError
+            raise LoadAttributeError, "JSON Parser Failed. - #{f}"
+          end
+        end
+      end
+
+      def merge_attributes(source, other = nil)
+        if source.class == Hash
+          merged = source.deep_merge(other)
+        elsif source.class == Array
+          if source.empty?
+            merged = {}
+          else
+            merged = source[0]
+            source.each {|s| merged.deep_merge!(s) }
+          end
+        end
+
+        merged
+      end
+
+      def create_tmp_nodes(filename, hash)
+        json = Base.jq(hash)
+        Base.write_tmp_nodes(filename) {|f| f.puts json }
+      end
+
+      def create_itamae_command(node_name, hash)
+        command = 'bundle exec itamae ssh'
+        command << if hash[:environments][:local_ipv4]
+                     " -h #{hash[:environments][:local_ipv4]}"
+                   else
+                     " -h #{hash[:environments][:hostname]}"
+                   end
+
+        command << " -u #{hash[:environments][:ssh_user]}"
+        command << " -p #{hash[:environments][:ssh_port]}"
+        command << " -i keys/#{hash[:environments][:ssh_key]}" unless hash[:environments][:ssh_key].nil?
+        command << " -j tmp-nodes/#{node_name}.json"
+
+        hash[:environments][:shell] = ENV['shell'] if ENV['shell']
+        command << if hash[:environments][:shell]
+                     " --shell=#{hash[:environments][:shell]}"
+                   else
+                     ' --shell=bash'
+                   end
+
+        command << ' --ask-password' unless hash[:environments][:ssh_password].nil?
+        command << ' --dry-run' if ENV['dry-run'] == 'true'
+        command << ' --log-level=debug' if ENV['debug'] == 'true'
+        command << ' --vagrant' if ENV['vagrant'] == 'true'
+        command
+      end
+
+      def create_spec_command(node_name, hash)
+        ENV['TARGET_HOST'] = if hash[:environments][:local_ipv4].nil?
+                               hash[:environments][:hostname]
+                             else
+                               hash[:environments][:local_ipv4]
+                             end
+
+        ENV['NODE_FILE'] = "tmp-nodes/#{node_name}.json"
+        ENV['SSH_PASSWORD'] = hash[:environments][:ssh_password]
+        ENV['SUDO_PASSWORD'] = hash[:environments][:sudo_password]
+        ENV['SSH_KEY'] = "keys/#{hash[:environments][:ssh_key]}"
+        ENV['SSH_USER'] = hash[:environments][:ssh_user]
+        ENV['SSH_PORT'] = hash[:environments][:ssh_port]
+
+        command = 'bundle exec rspec'
+        # ENV['vagrant'] TODO
+      end
+
+      def list_recipe_filepath(run_list)
+        recipes = []
+        run_list.each do |recipe|
+          target_list = Dir.glob("site-cookbooks/**/#{recipe.keys.join}/recipes/#{recipe.values.join}.rb")
+
+          raise LoadRecipeError, "#{recipe.to_a.join('::')} cookbook or recipe does not exist." if target_list.empty?
+
+          target_list.each do |target|
+            recipes << " #{target}"
+          end
+        end
+
+        recipes
+      end
+
+      def runner_display(raw_run_list, run_list, command)
+        run_list_str = run_list.map do |recipe|
+          if recipe.values.join == 'default'
+            recipe.keys.join
+          else
+            "#{recipe.keys.join}::#{recipe.values.join}"
+          end
+        end
+
+        ItamaeMitsurin.logger.color(:green) do
+          ItamaeMitsurin.logger.info "Run List is [#{raw_run_list.join(', ')}]"
+          ItamaeMitsurin.logger.info "Run List expands to [#{run_list_str.join(', ')}]"
+        end
+
+        ItamaeMitsurin.logger.color(:white) do
+          ItamaeMitsurin.logger.info command
+        end
+      end
+    end
+  end
+end

data/lib/itamae-mitsurin/mitsurin/creators/templates/project/site-cookbooks/_base/_base/recipes/default.rb

@@ -1,11 +0,0 @@
-ItamaeMitsurin.logger.color(:green) do
-  ItamaeMitsurin.logger.info "Environment: #{node[:environments][:set]}"
-  ItamaeMitsurin.file_logger.info "Environment: #{node[:environments][:set]}"
-end
-
-#Aws.config.update({
-#  region: "ap-northeast-1",
-#  credentials: Aws::Credentials.new(
-#    'key',
-#    'secret'
-#)})

data/lib/itamae-mitsurin/mitsurin/itamae_task.rb

@@ -1,150 +1,64 @@
-require 'itamae-mitsurin/mitsurin/task_base'
-include Rake::DSL if defined? Rake::DSL
+require 'itamae-mitsurin/mitsurin/base_task'
 
 module ItamaeMitsurin
   module Mitsurin
-    class ItamaeTask
+    class ItamaeTask < BaseTask
+      ItamaeMitsurin.logger.formatter.colored = true
+      task = ItamaeTask.new
 
       namespace :itamae do
-        Dir.glob("nodes/**/*.json").each do |node_file|
-          bname = File.basename(node_file, '.json')
+        all = []
 
+        Dir.glob('nodes/**/*.json').each do |node_file|
           begin
-            node_h = JSON.parse(File.read(node_file), symbolize_names: true)
-          rescue JSON::ParserError => e
-            puts e.class.to_s + ", " + e.backtrace[0].to_s
-            puts "Node load error, nodefile:#{node_file}, reason:#{e.message}"
+            node_name = File.basename(node_file, '.json')
+            node = task.load_node_attributes(node_file)
+            node_short = node[:environments][:hostname].split('.')[0]
+          rescue => e
+            ItamaeMitsurin.logger.error e.inspect
+            ItamaeMitsurin.logger.info "From node file: #{node_file}"
+            exit 2
           end
 
-          desc "Itamae to #{bname}"
-          task node_h[:environments][:hostname].split(".")[0] do
-            begin
-              recipes = []
-              TaskBase.get_roles(node_file).each do |role|
-                recipes << TaskBase.get_recipes(role)
-              end
-              TaskBase.get_node_recipes(node_file).each do |recipe|
-                recipes << recipe
-              end
-            rescue Exception => e
-              puts e.class.to_s + ", " + e.backtrace[0].to_s
-              puts "Node or role error, nodefile:#{node_file}, reason:#{e.message}"
-            else
-              recipes.flatten!
-            end
+          all << node_short
+          desc 'Itamae to all nodes'
+          task 'all' => all
 
-            # get env attr
+          desc "Itamae to #{node_name}"
+          task node_short do
             begin
-              env_set = node_h[:environments][:set]
-              raise "No environments set error" if env_set.nil?
-              env_h = JSON.parse(File.read("environments/#{env_set}.json"), symbolize_names: true)
-            rescue Exception => e
-              puts e.class.to_s + ", " + e.backtrace[0].to_s
-              puts "Node or environment load error, nodefile:#{node_file}, reason:#{e.message}"
-            end
-
-            # get recipes attr
-            recipe_attr_file = []
-            recipes.each do |recipe_h|
-              if recipe_h["#{recipe_h.keys.join}"] == "default"
-                recipe_attr_file.insert 0,
-                    Dir.glob("site-cookbooks/**/#{recipe_h.keys.join}/attributes/#{recipe_h["#{recipe_h.keys.join}"]}.json")
-              else
-                recipe_attr_file <<
-                    Dir.glob("site-cookbooks/**/#{recipe_h.keys.join}/attributes/#{recipe_h["#{recipe_h.keys.join}"]}.json")
-              end
-            end
-
-            recipe_attr_file.flatten!
-
-            # recipes attr other=env
-            recipe_env_h_a = []
-            recipe_attr_file.each do |file|
-              recipe_h = JSON.parse(File.read(file), symbolize_names: true)
-              recipe_env_h_a << recipe_h.deep_merge(env_h)
-            end
-
-            # recipe attr other=recipes_env
-            moto = recipe_env_h_a[0]
-            recipe_env_h_a.each {|hash| moto.deep_merge!(hash)}
-            recipe_env_h = moto
-
-            if recipe_env_h.nil?
-              # env attr other=node
-              node_env_h = env_h.deep_merge(node_h)
-              node_env_j = TaskBase.jq node_env_h
-              TaskBase.write_json(bname) {|file| file.puts node_env_j}
-            else
-              # recipe_env attr other=node
-              recipe_env_node_h = recipe_env_h.deep_merge(node_h)
-              recipe_env_node_j = TaskBase.jq recipe_env_node_h
-              TaskBase.write_json(bname) {|file| file.puts recipe_env_node_j}
-            end
-
-            recipes << {'_base' => 'default'}
-            node_property = JSON.parse(File.read("tmp-nodes/#{bname}.json"), symbolize_names: true)
-            node = node_property[:environments][:hostname]
-            ssh_user = node_property[:environments][:ssh_user]
-            ssh_password = node_property[:environments][:ssh_password]
-            sudo_password = node_property[:environments][:sudo_password]
-            ssh_port = node_property[:environments][:ssh_port]
-            ssh_key = node_property[:environments][:ssh_key]
-
-            ENV['TARGET_HOST'] = node
-            ENV['NODE_FILE'] = node_file
-            ENV['SSH_PASSWORD'] = ssh_password
-            ENV['SUDO_PASSWORD'] = sudo_password
-
-            command = "bundle exec itamae ssh"
-            command << " -h #{node}"
-            command << " -u #{ssh_user}"
-            command << " -p #{ssh_port}"
-            command << " -i keys/#{ssh_key}" unless ssh_key.nil?
-            command << " -j tmp-nodes/#{bname}.json"
-            command << " --shell=bash"
-            command << " --ask-password" unless ssh_password.nil?
-            command << " --dry-run" if ENV['dry-run'] == "true"
-            command << " -l debug" if ENV['debug'] == "true"
-            command << " -c logs/config/itamae_task.config"
-
-            # Pass to read the recipe command
-            command_recipe = []
-            recipes.each do |recipe_h|
-              target_recipe = "site-cookbooks/**/#{recipe_h.keys.join}/recipes/#{recipe_h[recipe_h.keys.join]}.rb"
-              if Dir.glob(target_recipe).empty?
-                raise "Recipe load error, nodefile: #{node_file}, reason: Does not exist " +
-                      recipe_h.keys.join + '::' +recipe_h.values.join
-              end
-              Dir.glob(target_recipe).join("\s").split.each do |target|
-                unless File.exists?(target)
-                  ex_recipe = recipe_h.to_s.gsub('=>', '::').gsub('"', '')
-                  raise "Recipe load error, nodefile:#{node_file}, reason: Does not exist #{ex_recipe}"
+              run_list = task.load_run_list(node_file)
+              environments = task.load_environments(node)
+              recipe_attributes_list = task.load_recipe_attributes(run_list)
+
+              merged_recipe = task.merge_attributes(recipe_attributes_list)
+              merged_environments = task.merge_attributes(merged_recipe, environments)
+              attributes = task.merge_attributes(merged_environments, node)
+              task.create_tmp_nodes(node_name, attributes)
+
+              command = task.create_itamae_command(node_name, attributes)
+              command_recipe = task.list_recipe_filepath(run_list)
+              command_recipe.sort_by! {|item| File.dirname(item) }
+              command << command_recipe.join
+
+              task.runner_display(attributes[:run_list], run_list, command)
+              st = system command
+              if st
+                ItamaeMitsurin.logger.color(:green) do
+                  ItamaeMitsurin.logger.info 'itamae_task is completed.'
                 end
-                command_recipe << " #{target}"
-              end
-            end
-
-            command_recipe.sort_by! {|item| File.dirname(item) }
-            command << command_recipe.join
-
-            puts TaskBase.hl.color(%!Run Itamae to "#{bname}"!, :red)
-            run_list_noti = []
-            command_recipe.each { |c_recipe|
-              unless c_recipe.split("/")[4].split(".")[0] == 'default'
-                run_list_noti << c_recipe.split("/")[2] + "::#{c_recipe.split("/")[4].split(".")[0]}"
               else
-                run_list_noti << c_recipe.split("/")[2]
+                ItamaeMitsurin.logger.error 'itamae_task is failed.'
+                exit 1
               end
-            }
-
-            puts TaskBase.hl.color(%!Run List to \"#{run_list_noti.uniq.join(", ")}\"!, :green)
-            puts TaskBase.hl.color(%!#{command}!, :white)
-            st = system command
-            exit 1 unless st
+            rescue => e
+              ItamaeMitsurin.logger.error e.inspect
+              ItamaeMitsurin.logger.info "From node file: #{node_file}"
+              exit 2
+            end
           end
         end
       end
-
     end
   end
 end