knife-topo 0.0.3

data/lib/chef/knife/topo_cookbook_create.rb

@@ -0,0 +1,178 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2014 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative 'topology_helper'
+require 'chef/knife/cookbook_create'
+
+class Chef
+  class Knife
+    class TopoCookbookCreate < Chef::Knife
+
+      deps do
+        Chef::Knife::CookbookCreate.load_deps
+      end
+
+      banner "knife topo cookbook create TOPOLOGY [ TOPOLOGY_FILE ] (options)"
+      
+      option :data_bag,
+      :short => '-D DATA_BAG',
+      :long => "--data-bag DATA_BAG",
+      :description => "The data bag the topologies are stored in"
+
+      # Make the base cookbook create options available on topo cookbook
+      self.options = (Chef::Knife::CookbookCreate.options).merge(self.options)
+
+      def initialize (args)
+        super
+        @cookbook_create_args  = initialize_cmd_args(args, [ 'cookbook', 'create' ])
+
+        # All called commands need to accept union of options
+        Chef::Knife::CookbookCreate.options = options
+
+      end
+
+      def run
+        if !@name_args[0]
+          show_usage
+          ui.fatal("You must specify the name of a topology")
+          exit 1
+        end
+ 
+        @bag_name = topo_bag_name(config[:data_bag])
+        @topo_name = @name_args[0]
+        @topo_file = @name_args[1]
+
+        # Get the topology data from either the file or the server
+        if @topo_file
+          topologies = load_topologies(@topo_file)
+          index = topologies.find_index{ |t| t['name'] == @topo_name}
+          unless index
+            ui.fatal("Topology #{@topo_name} was not found in topology file #{@topo_file}")
+            exit(1)
+          end
+          topo = topologies[index]
+        else
+          unless topo = load_from_server(@bag_name, @topo_name )
+            ui.fatal("Topology #{@bag_name}/#{@topo_name} does not exist on the server - use 'knife topo create' first")
+            exit(1)
+          end
+        end
+
+        # create the topology cookbooks
+        attribute_cookbooks = topo['cookbook_attributes']
+        @failed = []
+        succeeded = 0
+        if attribute_cookbooks && attribute_cookbooks.length > 0
+          attribute_cookbooks.each do |cookbook_spec|
+            run_create_cookbook(cookbook_spec['cookbook'])
+            create_attr_file(@cookbook_path, cookbook_spec['cookbook'],
+            cookbook_spec['filename'] + ".rb", cookbook_spec)
+            succeeded += 1
+          end
+        else
+          ui.info "No cookbook attributes found for topology #{@topo_name}"
+        end
+      end
+
+      private
+
+      include Chef::Knife::TopologyHelper
+
+      def run_create_cookbook(cookbook_name)
+        @cookbook_create_args[2] = cookbook_name
+        begin
+          command = run_cmd(Chef::Knife::CookbookCreate, @cookbook_create_args)
+        rescue Exception => e
+          raise if Chef::Config[:verbosity] == 2
+          @failed << cookbook_name
+          ui.warn "create of cookbook #{cookbook_name} exited with error"
+          humanize_exception(e)
+        end
+
+        # Store the cookbook path for use later
+        @cookbook_path = File.expand_path(Array(command.config[:cookbook_path]).first)
+        @copyright = command.config[:cookbook_copyright] || "YOUR_COMPANY_NAME"
+
+      end
+
+      def create_attr_file(dir, cookbook_name, filename, attrs)
+        
+        ui.info("** Creating attribute file #{filename}")
+        
+        open(File.join(dir, cookbook_name, "attributes", filename), "w") do |file|
+          file.puts <<-EOH
+#
+# THIS FILE IS GENERATED BY THE KNIFE TOPO PLUGIN - MANUAL CHANGES WILL BE OVERWRITTEN 
+#
+# Cookbook Name:: #{cookbook_name}
+# Attribute File:: #{filename}
+#
+# Copyright #{Time.now.year}, #{@copyright}
+#
+          EOH
+          
+          # Print out attribute line
+          def print_attr(file, lhs, value1)
+            if value1.is_a?(Hash)
+              value1.each do |key, value2|
+                print_attr(file, "#{lhs}['#{key}']", value2)
+              end                           
+            else
+              file.write "#{lhs} = \"#{value1}\"\n"
+            end
+          end
+          
+          # Print out attributes hashed by priority
+          def print_priority_attrs(file, attrs, indent=0)
+            %w(default force_default normal override force_override).each do |priority|
+              if attrs[priority]
+                lhs = ""
+                indent.times { |i| lhs += " " }
+                lhs += priority
+                print_attr(file, lhs, attrs[priority])
+              end
+            end
+          end
+          
+          # Print out qualified attributes
+          def print_qualified_attr(file, qualifier, qualifier_hash)
+            qualifier_hash.each do |qualifier_value, qualified_attrs|
+              file.puts "if node['topo']['#{qualifier}'] == \"#{qualifier_value}\""
+              print_priority_attrs(file, qualified_attrs, 2)
+              file.puts "end"
+            end
+          end
+          
+          # Process the attributes not needing qualification
+          print_priority_attrs(file, attrs)
+          file.puts
+          
+          # Process attributes that need to be qualified (must appear after the above in cookbook)
+          %w(node_type).each do |qualifier|
+            if attrs[qualifier]
+              file.puts "# Attributes for specific #{qualifier}"
+              print_qualified_attr(file, qualifier, attrs[qualifier])
+            end
+          end        
+          
+        end
+      end
+       
+    end
+  end
+end

data/lib/chef/knife/topo_cookbook_upload.rb

@@ -0,0 +1,84 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2014 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+require_relative 'topology_helper'
+
+## only require if not already defined (to prevent warning about already initialized constants)
+require 'chef/knife/cookbook_upload' if !defined? Chef::Knife::CookbookUpload
+
+class Chef
+  class Knife
+    class TopoCookbookUpload < Chef::Knife
+      
+      deps do
+        Chef::Knife::CookbookUpload.load_deps
+      end
+
+      banner "knife topo cookbook upload [ TOPOLOGY ] (options)"
+
+      option :data_bag,
+      :short => '-D DATA_BAG',
+      :long => "--data-bag DATA_BAG",
+      :description => "The data bag the topologies are stored in"
+
+      # Make called command options available
+      self.options = (Chef::Knife::CookbookUpload.options).merge(self.options)
+
+      def initialize (args)
+        super
+        @topo_upload_args  = initialize_cmd_args(args, [ 'cookbook', 'upload' ])
+
+        # All called commands need to accept union of options
+        Chef::Knife::CookbookUpload.options = options
+      end
+      
+      def run
+        if !@name_args[0]
+          show_usage
+          ui.fatal("You must specify the name of a topology")
+          exit 1
+        end
+
+        bag_name = topo_bag_name(config[:data_bag])
+        topo_name = @name_args[0]
+
+        # Load the topology data 
+        unless topo = load_from_file(bag_name, topo_name )
+          ui.fatal("Topology file #{topologies_path}/#{bag_name}/#{topo_name}.json not found - use 'knife topo import' first")
+          exit(1)
+        end
+        
+        # Run cookbook upload command on the topology cookbooks
+        if topo['cookbook_attributes'].length > 0
+          argPos = 2
+          topo['cookbook_attributes'].each do |entry|
+            @topo_upload_args[argPos] = entry['cookbook']
+            argPos += 1
+          end
+          run_cmd(Chef::Knife::CookbookUpload, @topo_upload_args)
+        else
+          ui.info("No cookbooks found for topology #{topo_name}.")
+        end                      
+      end
+
+      include Chef::Knife::TopologyHelper
+
+    end
+  end
+end

data/lib/chef/knife/topo_create.rb

@@ -0,0 +1,110 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2014 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+require_relative 'topology_helper'
+require_relative 'topo_bootstrap'
+require_relative 'topo_cookbook_upload'
+
+class Chef
+  class Knife
+    class TopoCreate < Chef::Knife
+      
+      deps do
+        Chef::Knife::TopoCookbookUpload.load_deps
+        Chef::Knife::TopoBootstrap.load_deps
+      end
+
+      banner "knife topo create TOPOLOGY (options)"
+
+      option :data_bag,
+      :short => '-D DATA_BAG',
+      :long => "--data-bag DATA_BAG",
+      :description => "The data bag the topologies are stored in"
+
+      option :bootstrap,
+      :long => "--bootstrap",
+      :description => "Whether to bootstrap newly created nodes",
+      :boolean => false
+
+      option :no_upload,
+      :long => "--no-upload",
+      :description => "Do not upload cookbooks",
+      :boolean => false
+            
+      # Make called command options available
+      opts = self.options
+      self.options = (Chef::Knife::TopoBootstrap.options).merge(Chef::Knife::TopoCookbookUpload.options)
+      self.options.merge!(opts)
+
+      def initialize (args)
+        super
+        @topo_bootstrap_args  = initialize_cmd_args(args, [ 'topo', 'bootstrap', @name_args[0] ])
+        @topo_upload_args  = initialize_cmd_args(args, [ 'topo', 'cookbook', 'upload', @name_args[0] ])
+
+        # All called commands need to accept union of options
+        Chef::Knife::TopoBootstrap.options = options
+        Chef::Knife::TopoCookbookUpload.options = options
+      end
+      
+      def run
+        if !@name_args[0]
+          show_usage
+          ui.fatal("You must specify the name of a topology")
+          exit 1
+        end
+
+        bag_name = topo_bag_name(config[:data_bag])
+        topo_name = @name_args[0]
+
+        # Load the topology data & create the topology bag
+        unless topo = load_from_file(bag_name, topo_name )
+          ui.fatal("Topology file #{topologies_path}/#{bag_name}/#{topo_name}.json not found - use 'knife topo import' first")
+          exit(1)
+        end
+        @data_bag = create_bag(bag_name)
+        
+        # Add topology item to the data bag on the server
+        begin
+          topo.create
+         rescue Net::HTTPServerException => e
+          raise unless e.to_s =~ /^409/
+          ui.confirm("Topology already exists - do you want to re-create it", true, false)
+        end
+
+        # make sure env and cookbooks are in place
+        check_chef_env(topo['chef_environment']) if topo['chef_environment']
+        upload_cookbooks(@topo_upload_args) if (!config[:no_upload])  
+          
+        # setup the actual nodes on the server
+        create_or_update_nodes(topo)
+        ui.info("Topology created")
+        
+        # if bootstrap is specified, run the bootstrap command
+        run_cmd(Chef::Knife::TopoBootstrap, @topo_bootstrap_args) if config[:bootstrap]
+        
+      end
+      
+
+
+
+      include Chef::Knife::TopologyHelper      
+
+    end
+  end
+end

data/lib/chef/knife/topo_export.rb

@@ -0,0 +1,137 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2014 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+
+require_relative 'topology_helper'
+
+# NOTE: This command exports to stdout - do not add ui messages (other than fatal) to this command
+
+class Chef
+  class Knife
+    class TopoExport < Chef::Knife
+
+      deps do
+      end
+
+      banner "knife topo export [ TOPOLOGY [ NODE ... ]] (options)"
+
+      option :data_bag,
+      :short => '-D DATA_BAG',
+      :long => "--data-bag DATA_BAG",
+      :description => "The data bag the topologies are stored in"
+
+      def run
+
+        @bag_name = topo_bag_name(config[:data_bag])
+
+        @topo_name = @name_args[0]
+        @node_names = @name_args[1..-1]
+        
+        if @topo_name
+          if topo = load_from_server(@bag_name, @topo_name)
+            export = topo.raw_data
+          else
+            export = empty_topology
+          end
+
+          # merge in data for nodes that user explicitly specified
+          @node_names.each do |node_name|
+            merge_node_properties!(export['nodes'], node_name)
+          end
+
+        else
+          # export all topologies
+          export = []
+          if dbag = load_from_server(@bag_name)
+            dbag.keys.each do |topo_name|
+               if topo = load_from_server(@bag_name, topo_name)
+                export << topo.raw_data
+              end
+            end
+          end
+        end
+
+        output(Chef::JSONCompat.to_json_pretty(export))
+      end
+
+      # give user a template to get started
+      def empty_topology
+        {
+          "id" => @topo_name || "topo1",
+          "name" => @topo_name || "topo1",
+          "chef_environment" => "_default",
+          "tags" => [ ],
+          "normal" => { },
+          "nodes" => [
+          "node1" => empty_node("node1")
+          ],
+          "cookbook_attributes" => [{
+            "cookbook" =>  @topo_name || "topo1",
+            "filename" => "topology"
+          }]
+        }
+      end
+
+      def empty_node(name)
+        {
+          "name" => name,
+          "run_list" => [],
+          "ssh_host" => name,
+          "ssh_port" => "22",
+          "normal" => {},
+          "tags" => []
+        }
+      end
+
+      # get actual node properties for export
+      def node_export (node_name)
+        begin
+          node = Chef::Node.load(node_name).to_hash
+        rescue Net::HTTPServerException => e
+          raise unless e.to_s =~ /^404/
+          node = empty_node(node_name)
+        end
+        props = %w{name chef_environment normal run_list recipes}
+        node_data = {}
+
+        props.each do |prop|
+          node_data[prop] = node[prop] if node.has_key?(prop)
+        end
+        
+        node_data[tags] = node_data[normal][tags]
+
+        node_data
+      end
+
+      # merge hash properties with the actual node properties
+      def merge_node_properties!(nodes, node_name)
+        # find out if the node is already in the array
+        node = nodes.find{|n| n["name"] == node_name }
+        if node
+          node.merge!(node_export(node_name))
+        else
+          nodes.push(node_export(node_name))
+        end
+      end
+
+      include Chef::Knife::TopologyHelper
+
+    end
+  end
+end

data/lib/chef/knife/topo_import.rb

@@ -0,0 +1,100 @@
+#
+# Author:: Christine Draper (<christine_draper@thirdwaveinsights.com>)
+# Copyright:: Copyright (c) 2014 ThirdWave Insights LLC
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/knife'
+
+require_relative 'topology_helper'
+require_relative 'topo_cookbook_create'
+
+class Chef
+  class Knife
+    class TopoImport < Chef::Knife
+
+      deps do
+        Chef::Knife::TopoCookbookCreate.load_deps
+      end
+
+      banner "knife topo import [ TOPOLOGY_FILE [ TOPOLOGY ... ]] (options)"
+      
+      option :data_bag,
+      :short => '-D DATA_BAG',
+      :long => "--data-bag DATA_BAG",
+      :description => "The data bag to store the topologies in"
+
+      # Make called command options available
+      self.options = Chef::Knife::TopoCookbookCreate.options.merge(self.options)
+      
+      def initialize (args)
+        super
+        @topo_cookbook_args = initialize_cmd_args(args, [ 'topo', 'cookbook', '', '' ])
+
+        # All called commands need to accept union of options
+        Chef::Knife::TopoCookbookCreate.options = options
+      end
+
+      def run
+        
+        # load data from the topologies file
+        topo_file = @name_args[0] || 'topology.json'
+        topologies = load_topologies(topo_file)
+        bag_name = topo_bag_name(config[:data_bag])
+        topo_names = @name_args[1..-1] if @name_args[1]
+        
+        # make sure the topology bag directory exists
+        path = File.join(topologies_path, bag_name)
+        FileUtils.mkdir_p(path)
+
+        topologies.each do |topo_data|
+        
+          topo_name = topo_data['name'] || topo_data['id']
+          topo_data['id'] ||= topo_name
+          
+          # check against specific topology list
+          if topo_names
+            if topo_names.include?(topo_name)
+              topo_names.delete(topo_name)
+            else
+              next
+            end
+          end
+          
+          # write the databag for this topology
+          path = File.join(topologies_path, bag_name, topo_name  + '.json')
+          File.open(path,"w") do |f|
+            f.write(Chef::JSONCompat.to_json_pretty(topo_data))
+            f.close()
+            ui.info "Imported topology #{topo_name} into  #{path}"
+          end
+          
+          # run topo cookbook to generate the cookbooks for this topology
+          @topo_cookbook_args[2] = topo_name
+          @topo_cookbook_args[3] = topo_file
+          run_cmd(Chef::Knife::TopoCookbookCreate, @topo_cookbook_args)
+          
+        end
+            
+        ui.info "Did not find topologies #{topo_names.join(', ')} in the exchange file"  if topo_names && topo_names.length > 0
+        ui.info "Import finished"
+        
+      end
+      
+      include Chef::Knife::TopologyHelper
+
+    end
+  end
+end