clc-promote 0.4.5 → 0.7.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c7e7e10fbd438abaff88aaf54464fe4aa92b228e
-  data.tar.gz: 9baa2d648939134acbe44770a1213fb546a767a3
+  metadata.gz: 028a2d20e003fc5841ed185d5da849033b2879b2
+  data.tar.gz: d1fe5f381fd2ee79759d23a0809b1a7770f1698e
 SHA512:
-  metadata.gz: f5d124b3c6dc141ee429b766363215069d9ff7e42438a3af7c3a10bedf94262496c6ab0c97ca123b5faaf01f6ff895ce84998cfbc875f4cb0b0915f744e4dd6e
-  data.tar.gz: 306e8fed3140ad1c747cafc32b8d1aab7730c303d1385464bab80f549801e0b8442fd369d8abe5280a0665fb25a3b55a866ad62ca16f2ba463cc1445dbb76275
+  metadata.gz: 8f3601f98c0eab1f4191f60e9bc22221b2207787157da24665f709865b25bc147e5d4af7c0dfebed73b05f2747f4088698fd1cfc2d3a778a68a81c7172645b80
+  data.tar.gz: 07000fd20e24894db386e22bd178ca8268ef0efcab71d71d9d1bde7c9b1cb05844f1f7a3c987acb6281795aedfce08285cda3e10e093dc394834ce006280c61f

data/README.md

@@ -1,11 +1,11 @@
 # CLC-Promote Gem
 Drives our CI/CD pipeline and provides functionality for:
-* Versioning cookbooks, data bags and environments
+* Versioning cookbooks, data bags, roles, and environments
 * Manages Cookbook version constraints within environments
 * Manages the uploading of chef artifacts from CI to the QA chef Server and eventually to the production chef server
 * Promotes one environment's cookbooks to another
 
-This functionality is exposed via a collectin of rake tasks used on the CI server and a knife plugin used to perform deployments.
+This functionality is exposed via a collection of rake tasks used on the CI server and a knife plugin used to perform deployments.
 
 ## Configuration
 CLC-Promote uses configuration settings to determine the values of:
@@ -41,6 +41,7 @@ Setting                   |
 :cookbook_directory       | Defaults to `#{root}/cookbooks`
 :environment_directory    | Defaults to `#{root}/environments`
 :data_bag_directory       | Defaults to `#{root}/data_bags`
+:role_directory           | Defaults to `#{root}/roles`
 :temp_directory           | Defaults to `/tmp/promote`
 :node_name                | **mandatory**
 :client_key               | **mandatory**
@@ -48,7 +49,7 @@ Setting                   |
 
 
 ## CLC-Promote Rake Tasks
-**Note:** See the [below section on versioning](/gems/clc-promote#How are version numbers generated) for details on how versin numbers are generated.
+**Note:** See the [below section on versioning](/gems/clc-promote#How are version numbers generated) for details on how version numbers are generated.
 
 ### Promote:version_cookbook
 Bumps the version of an individual cookbook.
@@ -62,6 +63,12 @@ Bumps the version of an individual environment file.
 ### Promote:version_environments
 Same as `Promote:version_environment` but iterates all environment files.
 
+### Promote:version_role
+Bumps the version of an individual role file.
+
+### Promote:version_roles
+Same as `Promote:version_role` but iterates all role files.
+
 ### Promote:version_data_bag
 Bumps the version of an individual databag entry.
 DEPRECATED (breaks encrypted data bags and vaults).
@@ -79,8 +86,11 @@ Uploads all cookbook versions of an environment to the chef server. **Note**: on
 ### Promote:upload_environment
 Uploads an environment file to the chef server
 
+### Promote:upload_roles
+Uploads all role files to the chef server
+
 ### Promote:upload_data_bags
-Uploads all data bage to the chef server
+Uploads all data bags to the chef server
 
 ### Promote: constrain_environment
 Given an environment and its environment cookbook, this task edits the environment file and creates cookbook constraints based on the `Berksfile.lock` of the environment cookbook.
@@ -95,7 +105,7 @@ Version numbers are based on the last version tag which forms the major and mino
 The knife promote command deploys an environment to production.
 
 ```
-knife promote environment [source environment] [target environment]
+knife promote environment SOURCE_ENVIRONMENT TARGET_ENVIRONMENT [ --data-bags LIST ]
 ```
 
 This command performs the following:
@@ -103,7 +113,7 @@ This command performs the following:
 1. Copies the cookbook version constraints from source to target
 2. Commits the constraints to version control
 3. Uploads the target environment to the QA chef server
-4. Downloads all data bags from QA
+4. Downloads all ```secrets_*``` data bags and all roles from QA.  Additional data bags can be specified on the command line.  Note that data bags containing chef-vault keys ( ```*_keys.json``` ) are always skipped.
 5. Performs a version diff on all cookbooks between the target environment and production chef server
-6. Downloads all cookbooks from qa that are not on the prod server
-7. uploads all new cookbooks, the target environment and databags to the production chef server.
+6. Downloads all cookbooks from QA that are not on the production server
+7. Uploads all new cookbooks, the target environment, databags and roles to the production chef server.

data/Rakefile

@@ -1,6 +1,9 @@
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
 
+$:.unshift(File.dirname(__FILE__) + '/lib')
+require 'promote/version'
+
 RSpec::Core::RakeTask.new(:test)
 
 task :default => [:test]

data/clc-promote.gemspec

@@ -17,8 +17,9 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
 
   s.add_runtime_dependency 'clc-git', '~> 1.2', '>= 1.2.8'
-  s.add_runtime_dependency 'berkshelf', '~> 3.2', '>= 3.2.2'
+  s.add_runtime_dependency 'berkshelf', '~> 3.2', '>= 3.2.3'
+  s.add_runtime_dependency 'highline', '~> 1.6', '>= 1.6.21'
 
   s.add_development_dependency 'rspec', '~> 3.0', '>= 3.0.0'
   s.add_development_dependency 'rake', '~> 10.3', '>= 10.3.2'
-end
+end

data/lib/chef/knife/promote.rb

@@ -6,6 +6,21 @@ module KnifePromote
 
     banner "knife promote environment SOURCE_ENVIRONMENT DESTINATION_ENVIRONMENT"
 
+    option :dry,
+      :long => "--dry-run",
+      :description => "Do not perform actual promote. Just report what would be done.",
+      :default => false
+
+    option :bags,
+      :long => "--data-bags VALUE",
+      :description => "A comma-separated list of data bags to be promoted (in addition to all secrets_* data bags)",
+      :default => ['secrets_*'],
+      :proc => Proc.new { |b|
+        b = "secrets_*,#{b}"
+        b.split(',')
+      }
+
+
     def run
       if @name_args.length < 2
         show_usage
@@ -16,40 +31,54 @@ module KnifePromote
       source_env = @name_args[0]
       dest_env = @name_args[1]
 
-      ui.info "#{source_env} is being promoted to #{dest_env}"
-      ui.confirm "Are you sure #{source_env} deserves this promotion"
-
-      config = Promote::Config.new({
+      promote_config = Promote::Config.new({
         :repo_root => File.expand_path("../",Chef::Config[:cookbook_path][0]),
         :node_name => Chef::Config[:node_name],
         :client_key => Chef::Config[:client_key],
-        :chef_server_url => Chef::Config[:chef_server_url]
+        :chef_server_url => Chef::Config[:chef_server_url],
+        :temp_directory => '/tmp/promote_staging',
+        :bags => config[:bags]
       })
       destination_config = Promote::Config.new({
-        :repo_root => [config.temp_directory],
+        :repo_root => promote_config.temp_directory,
         :node_name => Chef::Config[:knife][:promote_prod_user],
         :client_key => Chef::Config[:knife][:promote_prod_client_key],
         :chef_server_url => Chef::Config[:knife][:promote_prod_url]
       })
 
-      promoter = Promote::Promoter.new(config)
-      uploader = Promote::Uploader.new(config)
-      #Chef::Config[:verbosity] = 2
+      # Confirm local git repo is in sync with origin/master
+      source = "environments/#{source_env}.json"
+      dest = "environments/#{dest_env}.json"
+      r = Promote::GitRepo.new(promote_config.repo_root)
+      r.check_sync(source, dest)
 
-      promoter.promote_to(source_env, dest_env)
+      ui.info "*** #{source_env} is being promoted to #{dest_env}"
+      ui.confirm "Are you sure this is what you want to do"
 
-      ui.info "Commiting and pushing changes to #{dest_env} back to git"
-      env_file = File.join(config.environment_directory, "#{dest_env}.json")
-      repo = Promote::GitRepo.new(env_file)
-      repo.commit("promoted new constraints from #{source_env} to #{dest_env}", true)
+      promoter = Promote::Promoter.new(promote_config)
+      uploader = Promote::Uploader.new(promote_config)
+      Chef::Config[:verbosity] = 2
 
-      ui.info "uploading #{dest_env} to #{config.chef_server_url}"
+      # Locally mirror the environment constraints over
+      promoter.promote_to(source_env, dest_env, ui)
+
+      if !config[:dry]
+        ui.info "Committing and pushing changes to #{dest_env} back to git"
+        env_file = File.join(promote_config.environment_directory, "#{dest_env}.json")
+        repo = Promote::GitRepo.new(env_file)
+        repo.commit("promoted new constraints from #{source_env} to #{dest_env}", true)
+      end
+
+      # Uploads the environment to QA
+      ui.info "uploading #{dest_env} to #{promote_config.chef_server_url}"
       uploader.upload_environment(dest_env)
 
       promoter.stage_promotion(dest_env, destination_config, ui)
-      promoter.upload_to(dest_env, destination_config, ui)
 
-      ui.info "promotion complete. Congratulations #{dest_env} on a well deserved promotion!!"
+      if !config[:dry]
+        promoter.upload_to(dest_env, destination_config, ui)
+        ui.info "promotion complete. Congratulations #{dest_env} on a well deserved promotion!!"
+      end
     end
   end
 end

data/lib/kitchen/provisioner/environment.rb

@@ -18,7 +18,7 @@ module Kitchen
       private
 
       def create_node
-        node_file = File.join(instance.busser[:test_base_path], "nodes/#{instance.suite.name}.json")
+        node_file = File.join(instance.verifier[:test_base_path], "nodes/#{instance.suite.name}.json")
         if File.exist?(node_file)
           node = JSON.parse(File.read(node_file))
           node[:run_list] = config[:run_list]

data/lib/promote.rb

@@ -2,7 +2,8 @@ require 'promote/config'
 require 'promote/cookbook'
 require 'promote/environment_file'
 require 'promote/git_repo'
-require 'promote/uploader'
-require 'promote/versioner'
+require 'promote/node_finder'
 require 'promote/promoter'
+require 'promote/uploader'
 require 'promote/utils'
+require 'promote/versioner'

data/lib/promote/config.rb

@@ -2,13 +2,20 @@ module Promote
   class Config
     def initialize(options = {})
       options.each do |k, v|
-      	self.send("#{k}=", v) if self.respond_to?(k) 
+      	self.send("#{k}=", v) if self.respond_to?(k)
       end
       self.repo_root ||= Dir.pwd
       self.cookbook_directory ||= File.join(repo_root, "cookbooks")
       self.environment_directory ||= File.join(repo_root, "environments")
       self.data_bag_directory ||= File.join(repo_root, "data_bags")
+      self.role_directory ||= File.join(repo_root, "roles")
       self.temp_directory ||= "/tmp/promote"
+      self.bags ||= ['secrets_*']
+    end
+
+    def reset_temp_dir
+      FileUtils.rm_rf(temp_directory) if Dir.exist?(temp_directory)
+      Dir.mkdir(temp_directory)
     end
 
     def to_hash
@@ -27,9 +34,11 @@ module Promote
     attr_accessor :cookbook_directory
     attr_accessor :environment_directory
     attr_accessor :data_bag_directory
+    attr_accessor :role_directory
     attr_accessor :temp_directory
     attr_accessor :node_name
     attr_accessor :client_key
     attr_accessor :chef_server_url
+    attr_accessor :bags
   end
-end
+end

data/lib/promote/cookbook.rb

@@ -1,4 +1,5 @@
 require 'berkshelf'
+require 'digest/sha1'
 require 'json'
 
 module Promote
@@ -77,15 +78,26 @@ module Promote
       return false
     end
 
+    def dependency_hash(environment_cookbook_name)
+      cb_changes = sync_latest_app_cookbooks(environment_cookbook_name)
+      hash_src = ""
+      dependencies.each do | k,v |
+        hash_src << "#{k}::#{v}::"
+      end
+      Digest::SHA1.hexdigest(hash_src)
+    end
+
     def sync_latest_app_cookbooks(environment_cookbook_name)
       result = {}
       latest_server_cookbooks = Utils.chef_server_cookbooks(@config, 1)
       env_cookbook = Cookbook.new(environment_cookbook_name, @config)
-      env_cookbook.metadata_dependencies do |key|
-        latest_version = latest_server_cookbooks[key]['versions'][0]
-        if dependencies.keys.include?(key) && latest_version > dependencies[key].to_s
-          berksfile_update(key)
-          result[key] = latest_version
+      env_cookbook.metadata_dependencies.each do |key|
+        cb_key = key[0]
+        next if cb_key == @name || !latest_server_cookbooks.has_key?(cb_key)
+        latest_version = latest_server_cookbooks[cb_key]['versions'][0]['version']
+        if dependencies.keys.include?(cb_key) && latest_version > dependencies[cb_key].to_s
+          berksfile_update(cb_key)
+          result[cb_key] = latest_version
         end
       end
       result

data/lib/promote/git_repo.rb

@@ -1,3 +1,5 @@
+require 'git'
+
 module Promote
   class GitRepo
     def initialize(scope_path)
@@ -42,6 +44,25 @@ module Promote
       end
     end
 
+    def check_sync(source, dest)
+      warnings = []
+      if git.status.changed.to_h.keys.include? source
+        warnings << "File #{source} has uncommitted changes."
+      end
+      git.fetch
+      git.diff('master','origin/master').entries.each do |f|
+        if f.path == source || f.path == dest
+          warnings << "File #{f.path} is not synced with origin/master."
+        end
+      end
+      if warnings.count > 0
+        puts '*** Git sync issues: Fix the following issues and try again:'
+        warnings.each { |w| puts "* #{w}" }
+        exit 1
+      end
+    end
+
+
     attr_accessor :scope_path
 
     private
@@ -68,4 +89,4 @@ module Promote
       end
     end
   end
-end
+end

data/lib/promote/node_finder.rb

@@ -0,0 +1,18 @@
+module Promote
+  class NodeFinder
+    def initialize(query, config)
+      @query = query
+      @config = config
+      Chef::Config.reset
+      Chef::Config[:client_key] = config.client_key
+      Chef::Config[:chef_server_url] = config.chef_server_url
+      Chef::Config[:node_name] = config.node_name
+      @searcher = Chef::Search::Query.new
+    end
+
+    def search
+      results = @searcher.search(:node, @query)
+      results[0]
+    end
+  end
+end

data/lib/promote/promoter.rb

@@ -8,44 +8,68 @@ module Promote
       @config = config
     end
 
-    def promote_to(source_environment, destination_environment)
+    def promote_to(source_environment, destination_environment, ui = nil)
       source = EnvironmentFile.new(source_environment, config)
       dest = EnvironmentFile.new(destination_environment, config)
-      
+
+      if ui
+        ui.info "Cookbook constraints being promoted to #{destination_environment}"
+        ui.info source.cookbook_versions
+      end
+
       dest.write_cookbook_versions(source.cookbook_versions)
     end
 
+    def monitor_promotion(source_environment, destination_environments, probe_interval, ui = nil)
+      destination_environments.each do |env|
+        promote_to(source_environment, env, ui)        
+      end
+    end
+
     def stage_promotion(promote_environment, destination_config, ui)
       ui.info "Staging artifacts from #{config.chef_server_url} to #{config.temp_directory}"
-      FileUtils.rm_rf(config.temp_directory) if Dir.exist?(config.temp_directory)
-      Dir.mkdir(config.temp_directory)
+      config.reset_temp_dir
 
       deploy_file = "/environments/#{promote_environment}.json"
       ui.info "Downloading #{deploy_file}..."
       download_files(config.temp_directory, deploy_file)
+
       ui.info "Downloading data bags..."
-      download_files(config.temp_directory, "/data_bags/*")
+      config.bags.each do |bag|
+        download_files(config.temp_directory, "/data_bags/#{bag}")
+      end
+
+      ui.info "Downloading roles..."
+      download_files(config.temp_directory, "/roles")
 
-      ui.info "Downloading cookbooks..."
+      ui.info "Downloading cookbooks missing on #{destination_config.chef_server_url}..."
+      ui.info "Fetching all versions of cookbooks on #{destination_config.chef_server_url}..."
       server_versions = Promote::Utils.chef_server_cookbooks(destination_config)
+      ui.info "Fetching the environment file content of #{promote_environment}"
       env_content = EnvironmentFile.new(promote_environment, Config.new({:repo_root => config.temp_directory}))
       env_content.cookbook_versions.each do |k,v|
+        ui.info "#{k}-#{v}"
         if !server_versions.has_key?(k) || (server_versions[k]["versions"].select {|version| version['version'] == v}).empty?
-          ui.info "#{k}-#{v}"
+          ui.info "Cookbook missing. Downloading..."
           download_files(config.temp_directory, "/cookbooks/#{k}", v)
+        else
+          ui.info "Cookbook already exists."
         end
-      end      
+      end
     end
 
     def upload_to(promote_environment, destination_config, ui)
       ui.info "Uploading staged artifacts to #{destination_config.chef_server_url}"
       uploader = Uploader.new(destination_config)
+      Chef::Config[:verbosity] = 2
       ui.info("Uploading #{promote_environment} environment file...")
       uploader.upload_environment(promote_environment)
       ui.info("Uploading data bags...")
       uploader.upload_data_bags
+      ui.info("Uploading roles...")
+      uploader.upload_roles
       ui.info("Uploading cookbooks...")
-      uploader.upload_cookbook_directory(destination_config.cookbook_directory)
+      uploader.upload_cookbook_directory(destination_config.cookbook_directory, ui)
     end
 
     private
@@ -55,12 +79,13 @@ module Promote
       pattern = Chef::ChefFS::FilePattern.new(path)
       local = Chef::ChefFS::FileSystem::ChefRepositoryFileSystemRootDir.new(
         {
-          'environments' => ["#{local_root}/environments"], 
+          'environments' => ["#{local_root}/environments"],
           'data_bags' => ["#{local_root}/data_bags"],
+          'roles' => ["#{local_root}/roles"],
           'cookbooks' => ["#{local_root}/cookbooks"]
         }
       )
-      Chef::ChefFS::FileSystem.copy_to(pattern, fs_config.chef_fs, local, 1, Chef::Config)
+      Chef::ChefFS::FileSystem.copy_to(pattern, fs_config.chef_fs, local, nil, Chef::Config)
     end
   end
-end
+end