spiceweasel 1.2.0 → 2.0.0

data/lib/spiceweasel/clusters.rb

@@ -0,0 +1,53 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Spiceweasel
+  class Clusters
+
+    attr_reader :create, :delete
+
+    def initialize(clusters, cookbooks, environments, roles)
+      @create = Array.new
+      @delete = Array.new
+      if clusters
+        Spiceweasel::Log.debug("clusters: #{clusters}")
+        clusters.each do |cluster|
+          cluster_name = cluster.keys.first
+          Spiceweasel::Log.debug("cluster: '#{cluster_name}' '#{cluster[cluster_name]}'")
+          # add a tag to the Nodes
+          cluster[cluster_name].each do |node|
+            node_name = node.keys.first
+            run_list = node[node_name]['run_list'] || ''
+            options = node[node_name]['options'] || ''
+            # cluster tag is the cluster name + runlist
+            tag = " -j '{\"tags\":[\"#{cluster_name}+#{run_list.gsub(/[ ,\[\]:]/, '')}\"]}'"
+            Spiceweasel::Log.debug("cluster: #{cluster_name}:#{node_name}:tag:#{tag}")
+            #push the tag back on the options
+            node[node_name]['options'] = options + tag
+          end
+          Spiceweasel::Log.debug("cluster2: '#{cluster_name}' '#{cluster[cluster_name]}'")
+          # let's reuse the Nodes logic
+          nodes = Spiceweasel::Nodes.new(cluster[cluster_name], cookbooks, environments, roles)
+          @create.concat(nodes.create)
+          @delete.concat(nodes.delete)
+        end
+      end
+    end
+
+  end
+end

data/lib/spiceweasel/config.rb

@@ -0,0 +1,46 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'mixlib/config'
+
+module Spiceweasel
+  class Config
+    extend Mixlib::Config
+
+    debug false
+
+    # logging
+    log_level :info
+    log_location STDOUT
+
+    knife_options ''
+
+    # do we really need these?
+    delete false
+    execute false
+    extractjson false
+    extractlocal false
+    extractyaml false
+    help false
+    novalidation false
+    parallel false
+    rebuild false
+    siteinstall false
+
+  end
+end

data/lib/spiceweasel/cookbooks.rb

@@ -0,0 +1,105 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2011-2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/cookbook/metadata'
+
+module Spiceweasel
+  class Cookbooks
+
+    attr_reader :cookbook_list, :create, :delete
+
+    def initialize(cookbooks = [])
+      @create = Array.new
+      @delete = Array.new
+      @cookbook_list = Hash.new
+      @dependencies = Array.new
+      #validate each of the cookbooks specified in the manifest
+      if cookbooks
+        Spiceweasel::Log.debug("cookbooks: #{cookbooks}")
+        cookbooks.each do |cookbook|
+          name = cookbook.keys.first
+          if cookbook[name]
+            version = cookbook[name]['version']
+            options = cookbook[name]['options']
+          end
+          Spiceweasel::Log.debug("cookbook: #{name} #{version} #{options}")
+          if File.directory?("cookbooks")
+            if File.directory?("cookbooks/#{name}") #TODO use the name from metadata
+              validateMetadata(name,version) unless Spiceweasel::Config[:novalidation]
+            else
+              if Spiceweasel::Config[:siteinstall] #use knife cookbook site install
+                @create.push("knife cookbook#{Spiceweasel::Config[:knife_options]} site install #{name} #{version} #{options}")
+              else #use knife cookbook site download, untar and then remove the tarball
+                @create.push("knife cookbook#{Spiceweasel::Config[:knife_options]} site download #{name} #{version} --file cookbooks/#{name}.tgz #{options}")
+                @create.push("tar -C cookbooks/ -xf cookbooks/#{name}.tgz")
+                @create.push("rm -f cookbooks/#{name}.tgz")
+              end
+            end
+          elsif !Spiceweasel::Config[:novalidation]
+            STDERR.puts "ERROR: 'cookbooks' directory not found, unable to validate, download and load cookbooks"
+            exit(-1)
+          end
+          @create.push("knife cookbook#{Spiceweasel::Config[:knife_options]} upload #{name} #{options}")
+          @delete.push("knife cookbook#{Spiceweasel::Config[:knife_options]} delete #{name} #{version} -a -y")
+          @cookbook_list[name] = version #used for validation
+        end
+        validateDependencies() unless Spiceweasel::Config[:novalidation]
+      end
+    end
+
+    #check the metadata for versions and gather deps
+    def validateMetadata(cookbook,version)
+      #check metadata.rb for requested version
+      metadata_file = "cookbooks/#{cookbook}/metadata.rb"
+      metadata = Chef::Cookbook::Metadata.new
+      metadata.from_file(metadata_file)
+      Spiceweasel::Log.debug("validateMetadata: #{cookbook} #{metadata.name} #{metadata.version}")
+      # Should the cookbook directory match the name in the metadata?
+      if metadata.name.empty?
+        Spiceweasel::Log.warn("No cookbook name in the #{cookbook} metadata.rb.")
+      elsif cookbook != metadata.name
+        STDERR.puts "ERROR: Cookbook '#{cookbook}' does not match the name '#{metadata.name}' in #{cookbook}/metadata.rb."
+        exit(-1)
+      end
+      if version && metadata.version != version
+        STDERR.puts "ERROR: Invalid version '#{version}' of '#{cookbook}' requested, '#{metadata.version}' is already in the cookbooks directory."
+        exit(-1)
+      end
+      metadata.dependencies.each do |dependency|
+        Spiceweasel::Log.debug("cookbook #{cookbook} metadata dependency: #{dependency}")
+        @dependencies.push(dependency[0])
+      end
+    end
+
+    #compare the list of cookbook deps with those specified
+    def validateDependencies()
+      Spiceweasel::Log.debug("cookbook validateDependencies: '#{@dependencies}'")
+      @dependencies.each do |dep|
+        if !member?(dep)
+          STDERR.puts "ERROR: Cookbook dependency '#{dep}' is missing from the list of cookbooks in the manifest."
+          exit(-1)
+        end
+      end
+    end
+
+    def member?(cookbook)
+      cookbook_list.keys.include?(cookbook)
+    end
+
+  end
+end

data/lib/spiceweasel/data_bags.rb

@@ -0,0 +1,89 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2011-2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'json'
+
+module Spiceweasel
+  class DataBags
+
+    attr_reader :create, :delete
+
+    def initialize(data_bags = [])
+      @create = Array.new
+      @delete = Array.new
+      if data_bags
+        Spiceweasel::Log.debug("data bags: #{data_bags}")
+        data_bags.each do |data_bag|
+          db = data_bag.keys.first
+          #check directories
+          if !File.directory?("data_bags") && !Spiceweasel::Config[:novalidation]
+            STDERR.puts "ERROR: 'data_bags' directory not found, unable to validate or load data bag items"
+            exit(-1)
+          end
+          if !File.directory?("data_bags/#{db}") && !Spiceweasel::Config[:novalidation]
+            STDERR.puts "ERROR: 'data_bags/#{db}' directory not found, unable to validate or load data bag items"
+            exit(-1)
+          end
+          @create.push("knife data bag#{Spiceweasel::Config[:knife_options]} create #{db}")
+          @delete.push("knife data bag#{Spiceweasel::Config[:knife_options]} delete #{db} -y")
+          if data_bag[db]
+            items = data_bag[db]['items']
+            secret = data_bag[db]['secret']
+            if secret && !File.exists?(File.expand_path(secret)) && !Spiceweasel::Config[:novalidation]
+              STDERR.puts "ERROR: secret key #{secret} not found, unable to load encrypted data bags for data bag #{db}."
+              exit(-1)
+            end
+          end
+          items = [] if items.nil?
+          Spiceweasel::Log.debug("data bag: #{db} #{secret} #{items}")
+          items.each do |item|
+            Spiceweasel::Log.debug("data bag #{db} item: #{item}")
+            if item =~ /\*/ #wildcard support, will fail if directory not present
+              files = Dir.glob("data_bags/#{db}/*.json")
+              items.concat(files.collect {|x| x[x.rindex('/')+1..-6]})
+              Spiceweasel::Log.debug("found items '#{items}' for data bag: #{db}")
+              next
+            end
+            validateItem(db, item) unless Spiceweasel::Config[:novalidation]
+            if secret
+              @create.push("knife data bag#{Spiceweasel::Config[:knife_options]} from file #{db} #{item}.json --secret-file #{secret}")
+            else
+              @create.push("knife data bag#{Spiceweasel::Config[:knife_options]} from file #{db} #{item}.json")
+            end
+          end
+        end
+      end
+    end
+
+    #validate the item to be loaded
+    def validateItem(db, item)
+      if !File.exists?("data_bags/#{db}/#{item}.json")
+        STDERR.puts "ERROR: data bag '#{db}' item '#{item}' file 'data_bags/#{db}/#{item}.json' does not exist"
+        exit(-1)
+      end
+      f = File.read("data_bags/#{db}/#{item}.json")
+      itemfile = JSON.parse(f) #invalid JSON will throw a trace
+      #validate the id matches the file name
+      if !item.eql?(itemfile['id'])
+        STDERR.puts "ERROR: data bag '#{db}' item '#{item}' listed in the manifest does not match the id '#{itemfile['id']}' within the 'data_bags/#{db}/#{item}.json' file."
+        exit(-1)
+      end
+    end
+
+  end
+end

data/lib/spiceweasel/environments.rb

@@ -0,0 +1,103 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2011-2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'json'
+
+module Spiceweasel
+  class Environments
+
+    attr_reader :environment_list, :create, :delete
+
+    def initialize(environments = [], cookbooks = {})
+      @create = Array.new
+      @delete = Array.new
+      @environment_list = Array.new
+      if environments
+        Spiceweasel::Log.debug("environments: #{environments}")
+        environments.each do |env|
+          name = env.keys[0]
+          Spiceweasel::Log.debug("environment: #{name}")
+          if File.directory?("environments")
+            validate(name, cookbooks) unless Spiceweasel::Config[:novalidation]
+          elsif !Spiceweasel::Config[:novalidation]
+            STDERR.puts "'environments' directory not found, unable to validate or load environments"
+            exit(-1)
+          end
+          if File.exists?("environments/#{name}.json")
+            @create.push("knife environment#{Spiceweasel::Config[:knife_options]} from file #{name}.json")
+          else #assume no .json means they want .rb and catchall for misssing dir
+            @create.push("knife environment#{Spiceweasel::Config[:knife_options]} from file #{name}.rb")
+          end
+          @delete.push("knife environment#{Spiceweasel::Config[:knife_options]} delete #{name} -y")
+          @environment_list.push(name)
+        end
+      end
+    end
+
+    #validate the content of the environment file
+    def validate(environment, cookbooks)
+      #validate the environments passed in match the name of either the .rb or .json
+      if File.exists?("environments/#{environment}.rb")
+        #validate that the name inside the file matches
+        name = File.open("environments/#{environment}.rb").grep(/^name/)[0].split()[1].gsub(/"/,'').to_s
+        if !environment.eql?(name)
+          STDERR.puts "ERROR: Environment '#{environment}' listed in the manifest does not match the name '#{name}' within the environments/#{environment}.rb file."
+          exit(-1)
+        end
+        #validate the cookbooks exist if they're mentioned
+        envcookbooks = File.open("environments/#{environment}.rb").grep(/^cookbook /)
+        envcookbooks.each do |cb|
+          dep = cb.split()[1].gsub(/"/,'').gsub(/,/,'')
+          Spiceweasel::Log.debug("environment: '#{environment}' cookbook: '#{dep}'")
+          if !cookbooks.member?(dep)
+            STDERR.puts "ERROR: Cookbook dependency '#{dep}' from environment '#{environment}' is missing from the list of cookbooks in the manifest."
+            exit(-1)
+          end
+        end
+      elsif File.exists?("environments/#{environment}.json")
+        #load the json, don't symbolize since we don't need json_class
+        f = File.read("environments/#{environment}.json")
+        JSON.create_id = nil
+        envfile = JSON.parse(f, {:symbolize_names => false})
+        Spiceweasel::Log.debug("environment: '#{environment}' file: '#{envfile}'")
+        #validate that the name inside the file matches
+        Spiceweasel::Log.debug("environment: '#{environment}' name: '#{envfile['name']}'")
+        if !environment.eql?(envfile['name'])
+          STDERR.puts "ERROR: Environment '#{environment}' listed in the manifest does not match the name '#{envfile['name']}' within the 'environments/#{environment}.json' file."
+          exit(-1)
+        end
+        #validate the cookbooks exist if they're mentioned
+        envfile['cookbook_versions'].keys.each do |cb|
+          Spiceweasel::Log.debug("environment: '#{environment}' cookbook: '#{cb}'")
+          if !cookbooks.member?(cb.to_s)
+            STDERR.puts "ERROR: Cookbook dependency '#{cb}' from environment '#{environment}' is missing from the list of cookbooks in the manifest."
+            exit(-1)
+          end
+        end
+      else #environment is not here
+        STDERR.puts "ERROR: Invalid Environment '#{environment}' listed in the manifest but not found in the environments directory."
+        exit(-1)
+      end
+    end
+
+    def member?(environment)
+      environment_list.include?(environment)
+    end
+
+  end
+end

data/lib/spiceweasel/execute.rb

@@ -0,0 +1,42 @@
+#
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'mixlib/shellout'
+
+module Spiceweasel
+  class Execute
+
+    # run the commands passed in
+    def initialize(commands)
+      # for now we're shelling out
+      commands.each do | cmd |
+        knife = Mixlib::ShellOut.new(cmd)
+        # check for parallel? and eventually use threads
+        knife.run_command
+        puts cmd
+        puts knife.stdout
+        puts knife.stderr
+        Spiceweasel::Log.debug(cmd)
+        Spiceweasel::Log.debug(knife.stdout)
+        Spiceweasel::Log.fatal(knife.stderr) if !knife.stderr.empty?
+      end
+    end
+
+  end
+end
+

data/lib/spiceweasel/extract_local.rb

@@ -0,0 +1,130 @@
+#
+# Author:: Geoff Meakin
+# Author:: Matt Ray (<matt@opscode.com>)
+#
+# Copyright:: 2012, Opscode, Inc <legal@opscode.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/cookbook/metadata'
+
+module Spiceweasel
+  class ExtractLocal
+
+    def self.parse_objects
+      objects = {'cookbooks' => nil, 'roles' => nil, 'environments' => nil, 'data bags' => nil, 'nodes' => nil}
+
+      # COOKBOOKS
+      cookbooks = Array.new
+      Dir.glob('cookbooks/*').each do |cookbook_full_path|
+        metadata = Chef::Cookbook::Metadata.new
+        metadata.from_file("#{cookbook_full_path}/metadata.rb")
+        if metadata.name.empty?
+          Spiceweasel::Log.fatal("No cookbook name in the #{cookbook_full_path}/metadata.rb.")
+          exit(-1)
+        end
+        Spiceweasel::Log.debug("dir_ext: #{metadata.name} #{metadata.version}")
+        cookbooks.push(metadata)
+      end
+      cookbooks = self.order_cookbooks_by_dependency(cookbooks)
+      objects['cookbooks'] = cookbooks unless cookbooks.empty?
+
+      # ROLES
+      roles = []
+      Dir.glob("roles/*.{rb,json}").each do |role_full_path|
+        role = self.grab_name_from_path(role_full_path)
+        Spiceweasel::Log.debug("dir_ext: role: '#{role}'")
+        roles << {role => nil}
+      end
+      objects['roles'] = roles unless roles.nil?
+
+      # ENVIRONMENTS
+      environments = []
+      Dir.glob("environments/*.{rb,json}").each do |environment_full_path|
+        environment = self.grab_name_from_path(environment_full_path)
+        Spiceweasel::Log.debug("dir_ext: environment: '#{environment}'")
+        environments << {environment => nil}
+      end
+      objects['environments'] = environments unless environments.empty?
+
+      # DATA BAGS
+      data_bags = []
+      Dir.glob('data_bags/*').each do |data_bag_full_path|
+        data_bag = data_bag_full_path.split('/').last
+        Spiceweasel::Log.debug("dir_ext: data_bag: '#{data_bag}'")
+        data_bag_items = []
+        Dir.glob("#{data_bag_full_path}/*.{rb,json}").each do |data_bag_item_full_path|
+          Spiceweasel::Log.debug("dir_ext: data_bag: '#{data_bag}':'#{data_bag_item_full_path}'")
+          data_bag_items << self.grab_name_from_path(data_bag_item_full_path)
+        end if File.directory?(data_bag_full_path)
+        data_bags << {data_bag => {'items' => data_bag_items}}
+      end
+      objects['data bags'] = data_bags unless data_bags.empty?
+
+      # NODES
+      # TODO: Cant use this yet as node_list.rb doesnt support node from file syntax but expects the node info to be part of the objects passed in
+      # nodes = []
+      # Dir.glob("nodes/*.{rb,json}").each do |node_full_path|
+      #   node = self.grab_name_from_path(node_full_path)
+      #   nodes  << {node => nil}
+      # end
+      # objects['nodes'] = nodes unless nodes.empty?
+
+      objects
+    end
+
+    def self.grab_name_from_path(path)
+      name = path.split('/').last.split('.')
+      if name.length > 1
+        name.pop
+      end
+      name.join('.')
+    end
+
+    def self.order_cookbooks_by_dependency(cookbooks)
+      # Weak algorithm, not particularly elegant, ignores version info as unlikely to have two versions of a cookbook anyway
+      # We're going to find the cookbooks with their dependencies matched and keep going until all we have is unmatched deps
+      sorted_cookbooks = Array.new
+      unsorted_cookbooks = cookbooks
+      scount = 0
+      #keep looping until no more cookbooks are left or can't remove remainders
+      while unsorted_cookbooks.any? and scount < cookbooks.length
+        cookbook = unsorted_cookbooks.shift
+        Spiceweasel::Log.debug("dir_ext: cookbook.dependencies: '#{cookbook.name}' #{cookbook.dependencies}")
+        #if all the cookbook dependencies are in sorted_cookbooks
+        if sorted_cookbooks.eql?(sorted_cookbooks | cookbook.dependencies.collect {|x| x[0]})
+          sorted_cookbooks.push(cookbook.name)
+          scount = 0
+        else #put it back in the list
+          unsorted_cookbooks.push(cookbook)
+          scount = scount + 1
+        end
+        Spiceweasel::Log.debug("dir_ext: sorted_cookbooks: '#{sorted_cookbooks}' #{scount}")
+      end
+      if scount > 0
+        remainders = unsorted_cookbooks.collect {|x| x.name}
+        Spiceweasel::Log.debug("dir_ext: remainders: '#{remainders}'")
+        if Spiceweasel::Config[:novalidation] #stuff is missing, oh well
+          sorted_cookbooks.push(remainders).flatten!
+        else
+          deps = unsorted_cookbooks.collect {|x| x.dependencies.collect {|x| x[0]} - sorted_cookbooks}
+          STDERR.puts "ERROR: Dependencies not satisfied or circular dependencies in cookbook(s): #{remainders} depend(s) on #{deps}"
+          exit(-1)
+        end
+      end
+      #hack to get the format same as yaml/json parse
+      sorted_cookbooks.collect { |x| { x => nil } }
+    end
+  end
+end