knife-topo 1.1.2 → 2.0.1

data/lib/chef/knife/topo/processor/via_cookbook.rb

@@ -0,0 +1,118 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2015 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Converts data in a given format into V2 topo JSON format
+#
+require 'chef/knife/topo/processor'
+require 'chef/knife/topo/processor/via_cookbook_print'
+require 'chef/knife/topo/command_helper'
+require 'chef/knife/cookbook_upload' unless defined? Chef::Knife::CookbookUpload
+require 'chef/knife/cookbook_create'
+
+module KnifeTopo
+  # Class to provide context to execute knife topo helper methods
+  class KnifeHelper < Chef::Knife
+    include KnifeTopo::CommandHelper
+  end
+
+  # Process attributes via a cookbook
+  class ViaCookbookProcessor < KnifeTopo::Processor
+    attr_accessor :cookbook, :filename
+
+    include KnifeTopo::ViaCookbookPrint
+
+    def initialize(topo)
+      super
+      data = @topo['strategy_data'] || {}
+      @cookbook = data['cookbook'] || topo.topo_name
+      @filename = data['filename'] || 'topology'
+      @helper = KnifeHelper.new
+    end
+
+    register_processor('via_cookbook', name)
+
+    def generate_nodes
+      super
+    end
+
+    # generate attributes to cookbook
+    # context['cmd'] must be calling command
+    # context['cmd_args'] must be calling command's args
+    def generate_artifacts(context = {})
+      @cmd = context['cmd']
+      @cmd_args = context['cmd_args'] || []
+      @config = Chef::Config.merge!(@cmd.config)
+      return unless @cmd && cookbook_path
+      run_create_cookbook
+      create_attr_file(
+        cookbook_path,
+        cookbook_contents
+      )
+    end
+
+    def run_create_cookbook
+      create_args = @helper.initialize_cmd_args(
+        @cmd_args, @cmd.name_args, %w(cookbook create)
+      )
+      create_args[2] = @cookbook
+      # set options from calling command, so validation does not fail
+      Chef::Knife::CookbookCreate.options = @cmd.class.options
+      @helper.run_cmd(Chef::Knife::CookbookCreate, create_args)
+    rescue StandardError => e
+      raise if Chef::Config[:verbosity] == 2
+      @helper.ui.warn "Create of cookbook #{@cookbook} exited with error"
+      @helper.humanize_exception(e)
+    end
+
+    def create_attr_file(dir, contents)
+      @helper.ui.info("** Creating attribute file: #{@filename}")
+
+      name = @filename << '.rb' unless File.extname(@filename) == '.rb'
+      filepath = File.join(dir, @cookbook, 'attributes', name)
+      File.open(filepath, 'w') { |file| file.write(contents) }
+    end
+
+    def cookbook_path
+      paths = @config['cookbook_path']
+      return unless paths
+      paths.first
+    end
+
+    def upload_artifacts(context = {})
+      @cmd = context['cmd']
+      @cmd_args = context['cmd_args'] || []
+      return unless @cmd && !@cmd.config[:disable_upload]
+      run_upload_cookbook
+    end
+
+    def run_upload_cookbook
+      upload_args = @helper.initialize_cmd_args(
+        @cmd_args, @cmd.name_args, %w(cookbook upload)
+      )
+      upload_args[2] = @cookbook
+      # set options from calling command, so validation does not fail
+      Chef::Knife::CookbookUpload.options = @cmd.class.options
+      @helper.run_cmd(Chef::Knife::CookbookUpload, upload_args)
+    rescue StandardError => e
+      raise if Chef::Config[:verbosity] == 2
+      @helper.ui.warn "Upload of cookbook #{@cookbook} exited with error"
+      @helper.humanize_exception(e)
+    end
+  end
+end

data/lib/chef/knife/topo/processor/via_cookbook_print.rb

@@ -0,0 +1,97 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2015 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'chef/knife/topo/consts'
+
+module KnifeTopo
+  # Prints attribute file contents to string
+  module ViaCookbookPrint
+    def print_attr_header(contents, cookbook_name, filename)
+      copyright = @config['cookbook_copyright']
+      contents << "
+#
+# THIS FILE IS GENERATED BY KNIFE TOPO - MANUAL CHANGES WILL BE OVERWRITTEN
+#
+# Cookbook Name:: #{cookbook_name}
+# Attribute File:: #{filename}
+#
+"
+      return unless copyright
+      contents << "
+# Copyright (c) #{Time.now.year} #{copyright}
+#
+"
+    end
+
+    def print_attr(contents, lhs, value1)
+      if value1.is_a?(Hash)
+        value1.each do |key, value2|
+          print_attr(contents, "#{lhs}['#{key}']", value2)
+        end
+      else
+        ruby_str = value1.nil? ? 'nil' : Chef::JSONCompat.to_json(value1)
+        contents << "#{lhs} = #{ruby_str}\n"
+      end
+    end
+
+    def print_attrs(contents, attrs, indent = 0)
+      return unless attrs
+      KnifeTopo::PRIORITIES.each do |priority|
+        next unless attrs[priority]
+        lhs = ''
+        indent.times { lhs << ' ' }
+        lhs << priority
+        print_attr(contents, lhs, attrs[priority])
+      end
+    end
+
+    def print_attrs_for_node(contents, n)
+      if n['node_type']
+        print_node_type_attr_header(contents, n['node_type'])
+      else
+        print_node_name_attr_header(contents, n['name'])
+      end
+      print_attrs(contents, n, 2)
+      contents << "end\n"
+    end
+
+    def print_node_type_attr_header(contents, node_type)
+      contents << "# Attributes for node type #{node_type}\n"
+      contents << "if node['topo'] && node['topo']['node_type']" \
+       " == '#{node_type}'\n"
+    end
+
+    def print_node_name_attr_header(contents, node_name)
+      contents << "# Attributes for node name #{node_name}\n"
+      contents << "if node.name == '#{node_name}'\n"
+    end
+
+    def cookbook_contents
+      contents = ''
+      print_attr_header(contents, @cookbook, @filename)
+      print_attrs(contents, @topo['attributes'])
+      @topo.nodes.each do |n|
+        print_attrs_for_node(contents, n)
+      end
+      contents
+    end
+
+    def copyright
+      Chef::Config.copyright || @cmd.config['copyright']
+    end
+  end
+end

data/lib/chef/knife/topo/version.rb

@@ -1,5 +1,4 @@
-module Knife
-  module Topo
-    VERSION = "1.1.2"
-  end
+# version
+module KnifeTopo
+  VERSION = '2.0.1'
 end

data/lib/chef/knife/topo_bootstrap.rb

@@ -16,99 +16,127 @@
 # limitations under the License.
 #
 
-
-require_relative 'topology_helper'
+require 'chef/knife/topo/bootstrap_helper'
+require 'chef/knife/topo/loader'
 require 'chef/knife/bootstrap'
 
-class Chef
-  class Knife
-    class TopoBootstrap < Chef::Knife
-
-      deps do
-         Chef::Knife::Bootstrap.load_deps
-      end
-
-      banner "knife topo bootstrap TOPOLOGY (options)"
-
-      option :data_bag,
-      :short => '-D DATA_BAG',
-      :long => "--data-bag DATA_BAG",
-      :description => "The data bag the topologies are stored in"
-      
-      option :overwrite,
-      :long => "--overwrite",
-      :description => "Whether to overwrite existing nodes",
-      :boolean => true
+module KnifeTopo
+  # knife topo bootstrap
+  class TopoBootstrap < Chef::Knife
+    deps do
+      require 'chef/knife/topo/processor'
+    end
 
-      # Make the base bootstrap options available on topo bootstrap
-      self.options = (Chef::Knife::Bootstrap.options).merge(self.options)
+    include KnifeTopo::BootstrapHelper
+    include KnifeTopo::Loader
+
+    banner 'knife topo bootstrap TOPOLOGY (options)'
+
+    option(
+      :data_bag,
+      short: '-D DATA_BAG',
+      long: '--data-bag DATA_BAG',
+      description: 'The data bag the topologies are stored in'
+    )
+
+    option(
+      :overwrite,
+      long: '--overwrite',
+      description: 'Whether to overwrite existing nodes',
+      boolean: true
+    )
+
+    # Make the base bootstrap options available on topo bootstrap
+    self.options = (Chef::Knife::Bootstrap.options).merge(TopoBootstrap.options)
+
+    attr_accessor :msgs, :results
+
+    def bootstrap_msgs
+      {
+        bootstrapped: 'Bootstrapped %{num} nodes [ %{list} ]',
+        skipped: 'Unexpected error',
+        skipped_ssh: 'Did not bootstrap %{num} nodes [ %{list} ] ' \
+          'because they do not have an ssh_host',
+        existed: 'Did not bootstrap %{num} nodes [ %{list} ] because '\
+          "they already exist.\n"\
+          "Specify --overwrite to re-bootstrap existing nodes. \n",
+        failed: '%{num} nodes [ %{list} ] failed to bootstrap due to errors'
+      }
+    end
 
-      def initialize (args)
-        super
-        @bootstrap_args = initialize_cmd_args(args, [ 'bootstrap', '' ])
+    def initialize(args)
+      super
+      @bootstrap_args = initialize_cmd_args(args, name_args, ['bootstrap', ''])
+      @results = {
+        bootstrapped: [], skipped: [], skipped_ssh: [], existed: [], failed: []
+      }
+      @msgs = bootstrap_msgs
+      @bootstrap = true
+
+      # All called commands need to accept union of options
+      Chef::Knife::Bootstrap.options = options
+    end
 
-        # All called commands need to accept union of options
-        Chef::Knife::Bootstrap.options = options
+    def run
+      validate_args
 
+      # load and bootstrap each node that has a ssh_host
+      @topo = load_topo_from_server_or_exit(@topo_name)
+      @processor = KnifeTopo::Processor.for_topo(@topo)
+      nodes = @processor.generate_nodes
+      nodes.each do |node_data|
+        node_bootstrap(node_data)
       end
 
-      def run
-        if !@name_args[0]
-          show_usage
-          ui.fatal("You must specify the name of a topology")
-          exit 1
-        end
-
-        @bag_name = topo_bag_name(config[:data_bag])
-        @topo_name = @name_args[0]
+      report
+    end
 
-        # get the node names for the topology
-        unless topo = load_from_server(@bag_name, @topo_name )
-          ui.fatal("Topology #{@bag_name}/#{@topo_name} does not exist on the server - use 'knife topo create' first")
-          exit(1)
-        end
+    def validate_args
+      unless @name_args[0]
+        show_usage
+        ui.fatal('You must specify the name of a topology')
+        exit 1
+      end
+      @topo_name = @name_args[0]
+    end
 
-        # load and bootstrap each node that has a ssh_host
-        nodes = merge_topo_properties(topo['nodes'], topo)
-          
-        bootstrapped = []
-        skipped = []
-        existed = []
-        failed = [] 
-          
-        if nodes.length > 0
-          nodes.each do |node_data|
-            node_name = node_data['name']
-            exists = resource_exists?("nodes/#{node_name}") 
-            if(node_data['ssh_host'] && (config[:overwrite] || !exists))
-              if run_bootstrap(node_data, @bootstrap_args, exists)
-                bootstrapped << node_name
-              else
-                failed << node_name
-              end
-            else
-              if(exists)
-                existed << node_name
-              else
-                skipped << node_name
-              end
-            end
-          end
-          ui.info("Bootstrapped #{bootstrapped.length} nodes [ #{bootstrapped.join(', ')} ]")
-          ui.info("Skipped #{skipped.length} nodes [ #{skipped.join(', ')} ] because they had no ssh_host information") if skipped.length > 0
-          if existed.length > 0
-            ui.info("Skipped #{existed.length} nodes [ #{existed.join(', ')} ] because they already exist. " +
-              "Specify --overwrite to re-bootstrap existing nodes. " +
-              "If you are using Chef Vault, you may need to use --bootstrap-vault options in this case.")
-          end
-          ui.warn("#{failed.length} nodes [ #{failed.join(', ')} ] failed to bootstrap due to errors") if failed.length > 0
+    # rubocop:disable Metrics/MethodLength
+    def node_bootstrap(node_data)
+      node_name = node_data['name']
+      state = :skipped_ssh
+      if node_data['ssh_host']
+        exists = resource_exists?("nodes/#{node_name}")
+        if config[:overwrite] || !exists
+          success = run_bootstrap(node_data, @bootstrap_args, exists)
+          state = success ? :bootstrapped : :failed
         else
-          ui.info "No nodes found for topology #{display_name(topo)}"
+          state = :existed
         end
       end
+      @results[state] << node_name
+      success
+    end
+    # rubocop:enable Metrics/MethodLength
+
+    # Report is used by create, update and bootstrap commands
+    def report
+      if @topo['nodes'].length > 0
+        report_msg(:bootstrapped, :info, false) if @bootstrap
+        report_msg(:skipped, :info, true)
+        report_msg(:skipped_ssh, :info, true)
+        report_msg(:existed, :info, true)
+        report_msg(:failed, :warn, true) if @bootstrap
+      else
+        ui.info 'No nodes found'
+      end
+      ui.info("Topology: #{@topo.display_info}")
+    end
 
-      include Chef::Knife::TopologyHelper
-
+    def report_msg(state, level, only_non_zero = true)
+      nodes = @results[state]
+      return if only_non_zero && nodes.length == 0
+      ui.send(level, @msgs[state] %
+        { num: nodes.length, list: nodes.join(', ') })
     end
   end
 end

data/lib/chef/knife/topo_cookbook_create.rb

@@ -16,161 +16,58 @@
 # limitations under the License.
 #
 
-require_relative 'topology_helper'
+require 'chef/knife/topo/loader'
 require 'chef/knife/cookbook_create'
 
-class Chef
-  class Knife
-    class TopoCookbookCreate < Chef::Knife
-
-      deps do
-        Chef::Knife::CookbookCreate.load_deps
-      end
-
-      banner "knife topo cookbook create TOPOLOGY [ TOPOLOGY_FILE ] (options)"
-      
-      option :data_bag,
-      :short => '-D DATA_BAG',
-      :long => "--data-bag DATA_BAG",
-      :description => "The data bag the topologies are stored in"
-
-      # Make the base cookbook create options available on topo cookbook
-      self.options = (Chef::Knife::CookbookCreate.options).merge(self.options)
-
-      def initialize (args)
-        super
-        @cookbook_create_args  = initialize_cmd_args(args, [ 'cookbook', 'create' ])
-
-        # All called commands need to accept union of options
-        Chef::Knife::CookbookCreate.options = options
-
-      end
+module KnifeTopo
+  # knife topo cookbook create
+  class TopoCookbookCreate < Chef::Knife
+    deps do
+      require 'chef/knife/topo/processor'
+    end
 
-      def run
-        if !@name_args[0]
-          show_usage
-          ui.fatal("You must specify the name of a topology")
-          exit 1
-        end
- 
-        @bag_name = topo_bag_name(config[:data_bag])
-        @topo_name = @name_args[0]
-        @topo_file = @name_args[1]
+    banner 'knife topo cookbook create TOPOLOGY_FILE (options)'
 
-        # Get the topology data from either the file or the server
-        if @topo_file
-          topologies = load_topologies(@topo_file)
-          index = topologies.find_index{ |t| t['name'] == @topo_name}
-          unless index
-            ui.fatal("Topology #{@topo_name} was not found in topology file #{@topo_file}")
-            exit(1)
-          end
-          topo = topologies[index]
-        else
-          unless topo = load_from_server(@bag_name, @topo_name )
-            ui.fatal("Topology #{@bag_name}/#{@topo_name} does not exist on the server - use 'knife topo create' first")
-            exit(1)
-          end
-        end
+    option(
+      :data_bag,
+      short: '-D DATA_BAG',
+      long: '--data-bag DATA_BAG',
+      description: 'The data bag the topologies are stored in'
+    )
 
-        # create the topology cookbooks
-        attribute_cookbooks = topo['cookbook_attributes']
-        cookbook_names = []
-        if attribute_cookbooks && attribute_cookbooks.length > 0
-          attribute_cookbooks.each do |cookbook_spec|
-            cookbook_name = cookbook_spec['cookbook']
-            run_create_cookbook(cookbook_name) unless cookbook_names.include?(cookbook_name)
-            cookbook_names << cookbook_name
-            create_attr_file(@cookbook_path, cookbook_name,
-            cookbook_spec['filename'] + ".rb", cookbook_spec)
-          end
-        else
-          ui.info "No topology cookbook has been specified for topology #{@topo_name}"
-        end
-      end
+    # Make the base cookbook create options available on topo cookbook
+    self.options = Chef::Knife::CookbookCreate.options.merge(
+      TopoCookbookCreate.options)
 
-      private
+    include KnifeTopo::Loader
 
-      include Chef::Knife::TopologyHelper
+    def initialize(args)
+      super
+      @args  = args
+    end
 
-      def run_create_cookbook(cookbook_name)
-        @cookbook_create_args[2] = cookbook_name
-        begin
-          command = run_cmd(Chef::Knife::CookbookCreate, @cookbook_create_args)
-        rescue Exception => e
-          raise if Chef::Config[:verbosity] == 2
-          ui.warn "Create of cookbook #{cookbook_name} exited with error"
-          humanize_exception(e)
-        end
+    def run
+      validate_args
 
-        # Store the cookbook path for use later
-        @cookbook_path = File.expand_path(Array(command.config[:cookbook_path]).first)
-        @copyright = command.config[:cookbook_copyright] || "YOUR_COMPANY_NAME"
+      @topo = load_topo_from_file_or_exit(@topo_file)
+      @processor = KnifeTopo::Processor.for_topo(@topo)
+      do_create_artifacts
+    end
 
+    def validate_args
+      unless @name_args[0]
+        show_usage
+        ui.fatal('You must specify a topology JSON file')
+        exit 1
       end
+      @topo_file = @name_args[0]
+    end
 
-      def create_attr_file(dir, cookbook_name, filename, attrs)
-        
-        ui.info("** Creating attribute file #{filename}")
-        
-        open(File.join(dir, cookbook_name, "attributes", filename), "w") do |file|
-          file.puts <<-EOH
-#
-# THIS FILE IS GENERATED BY THE KNIFE TOPO PLUGIN - MANUAL CHANGES WILL BE OVERWRITTEN 
-#
-# Cookbook Name:: #{cookbook_name}
-# Attribute File:: #{filename}
-#
-# Copyright #{Time.now.year}, #{@copyright}
-#
-          EOH
-          
-          # Print out attribute line
-          def print_attr(file, lhs, value1)
-            if value1.is_a?(Hash)
-              value1.each do |key, value2|
-                print_attr(file, "#{lhs}['#{key}']", value2)
-              end                           
-            else
-              rubyString = (value1 == nil) ? "nil" : Chef::JSONCompat.to_json(value1);
-              file.write "#{lhs} = " + rubyString + " \n"
-            end
-          end
-          
-          # Print out attributes hashed by priority
-          def print_priority_attrs(file, attrs, indent=0)
-            %w(default force_default normal override force_override).each do |priority|
-              if attrs[priority]
-                lhs = ""
-                indent.times { |i| lhs += " " }
-                lhs += priority
-                print_attr(file, lhs, attrs[priority])
-              end
-            end
-          end
-          
-          # Print out qualified attributes
-          def print_qualified_attr(file, qualifier_hash)
-            file.puts "if node['topo'] && node['topo']['#{qualifier_hash['qualifier']}'] == \"#{qualifier_hash['value']}\""
-            print_priority_attrs(file, qualifier_hash, 2)
-            file.puts "end"
-          end
-          
-          # Process the attributes not needing qualification
-          print_priority_attrs(file, attrs)
-          file.puts
-          
-          # Process attributes that need to be qualified
-          if attrs['conditional']
-            attrs['conditional'].each do |qualified_attrs|
-              file.puts "# Attributes for specific #{qualified_attrs['qualifier']}"
-              print_qualified_attr(file, qualified_attrs)
-            end
-          end        
-          
-        end
-      end
-       
+    def do_create_artifacts
+      @processor.generate_artifacts(
+        'cmd_args' => @args,
+        'cmd' => self
+      )
     end
   end
-end
+end