fhcap-cli 0.4.13 → 0.4.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 78c7eb91b57300a3bde0b17a9e658c95daf0ae28
-  data.tar.gz: e53cae790a8184c7b0f3d39bc9b8b9f0196bd938
+  metadata.gz: df6384b3f0a956ef606c3ce4543bdf5416501505
+  data.tar.gz: e468e58706453980ecde19e48309ff3f7eb0d0b8
 SHA512:
-  metadata.gz: 3642959bb379209b96a820250ea2aaf16d142d894ac195cb3ec9261cd79a25a791cb57b3b809d92335ad0e43ae45c55f9a8a959cf4ed98d48e01778f8768b229
-  data.tar.gz: e4343b8c5346771acb9386a1d192fe36ba5659c8126381e412e3af8429614ac7019398b9fd7951d1c7ab3b01c23a6f5271eccf3488f11afb249b5760737533d8
+  metadata.gz: 32bbd3b888485117124bb672a68018f0a71212e799fe22ee94b0a6dbcb08654c60cefdb4c55734ded424171afc11fb2e49da49df773ad96e3a244c2cc20e2d82
+  data.tar.gz: cf572600c4a85f9e1f94c5711c9a40d1f1355f34831991ec6b6852493fdbf579c211711549db6b21e2580608a13ade2787cc915637b57477d4eaac9bae814918

data/CHANGELOG.md

@@ -1,6 +1,18 @@
-
 ## Unreleased
 
+## 0.4.14
+
+* [RHMAP-16161] - Adding S3 support for RHMAP component validation as part of list_artifacts task
+* [RHMAP-8094]  - Add cookbook check-bumps task
+* [RHMAP-16470] - Added cookbook update attribute task
+* [RHMAP-14096] - Adding platform version bump task
+* [RHMAP-15648] - Add Jenkinsfile to repository for Wendy Jenkins builds
+* [RHMAP-16052] - Allow updating clusters-dir option during repo add
+* [RHMAP-16052] - Add empty-config and knife-dir option to setup task.
+* [RHMAP-15292] - Renaming OSCP multi node cluster templates to ocp-multi and ocp-multi-infra
+* [RHMAP-15292] - Adding support for AWS Elastic File System
+* [RHMAP-15112] - Add 'dirty' option to cookbook update task. Allows manipulating cookbooks that have uncommited changes locally
+
 ## 0.4.13
 
 * [RHMAP-15127] - Add meta only option to cluster info task

data/Jenkinsfile

@@ -0,0 +1,26 @@
+#!groovy
+
+// https://github.com/feedhenry/fh-pipeline-library
+@Library('fh-pipeline-library') _
+
+node("ruby") {
+
+    step([$class: 'WsCleanup'])
+
+    stage('Checkout') {
+      checkout scm
+    }
+
+    stage('Install Dependencies') {
+        sh 'bundle install --binstubs'
+    }
+
+    stage('Unit Tests') {
+        sh 'bin/rspec --format documentation --color'
+    }
+
+    stage('Build') {
+        sh 'bin/rake build'
+        archiveArtifacts 'pkg/fhcap-cli-*.gem'
+    }
+}

data/lib/fhcap/cli.rb

@@ -20,6 +20,8 @@ module Fhcap
 
       method_option 'repos-dir', :type => :string, :desc => "Local directory where configured repos will be checked out"
       method_option 'fh-src-dir', :type => :string, :desc => "Local directory where FH component src repos are checked out"
+      method_option 'knife-dir', :type => :string, :desc => "Local directory where knife configuration will be stored"
+      method_option 'empty-config', :type => :boolean, :default => false, :desc => "Create an empty fhcap config i.e. No repos or knife config is pre configured"
 
       desc "setup", "Setup the FHCAP CLI"
 

data/lib/fhcap/component.rb

@@ -28,7 +28,7 @@ module Fhcap
         Fhcap::CLI::Component.new.invoke(:update_config, [component], {force: options[:force]}) if options['update-config']
 
         #3. Update cookbook
-        Fhcap::CLI::Cookbook.new.invoke(:update, [], {cookbooks: [artifact_cookbook], level: options['level']}) if options['update-cookbook']
+        Fhcap::CLI::Cookbook.new.invoke(:update, [], {dirty: true, level: options['level']}) if options['update-cookbook']
 
       end
 
@@ -67,11 +67,11 @@ module Fhcap
 
       method_option 'format', :type => :string, :default => 'table', :desc => "Format to output data in", :enum => %w{table json}
 
-      def list
+      def list_artifacts
         require "fhcap/tasks/chef/cookbook/list_artifacts"
         Tasks::Chef::Cookbook::ListArtifacts.new(task_options(options.dup)).run
       end
 
     end
   end
-end
+end

data/lib/fhcap/config.rb

@@ -98,6 +98,18 @@ module Fhcap
       }
     end
 
+    def fhcap_empty_config
+      @fhcap_config ||= {
+          :repos_dir => File.join(default_dir, 'repos'),
+          :fh_src_dir => File.join(default_dir, 'repos'),
+          :repos => {
+          },
+          :knife_dir => File.join(default_dir, '.chef'),
+          :knife => {
+          }
+      }
+    end
+
     private
 
     def symbolize_keys_deep(h)

data/lib/fhcap/cookbook.rb

@@ -5,8 +5,11 @@ module Fhcap
   module CLI
     class Cookbook < Fhcap::ThorBase
 
+      include Fhcap::ReposHelper
+
       add_shared_option :cookbooks, :aliases => "-c", :desc => "Apply to these Cookbooks", :type => :array
       add_shared_option :modified, :aliases => "-m", :desc => "Apply only to modified Cookbooks", :type => :boolean, :required => false, :default => false
+      add_shared_option :dirty, :desc => "Apply only to cookbooks with uncommitted local changes", :type => :boolean, :required => false, :default => false
 
       desc "update", "Update a cookbook, defaults to all cookbooks."
       long_desc <<-LONGDESC
@@ -31,7 +34,7 @@ module Fhcap
       fhcap cookbook update -c feedhenry_common -v 1.2.3
       LONGDESC
 
-      shared_options :cookbooks, :modified
+      shared_options :cookbooks, :modified, :dirty
 
       method_option 'level', :type => :string, :aliases => "-l", :desc => "Cookbook Bump level", :default => 'minor', :enum => %w{major minor patch}
       method_option 'version', :type => :string, :aliases => "-v", :desc => "Version to set manually"
@@ -47,6 +50,26 @@ module Fhcap
         end
       end
 
+      desc "check_bumps", "Check modified cookbook versions are bumped as expected"
+      long_desc <<-LONGDESC
+      Checks that all modified cookbook versions are bumped as expected against the base-branch
+
+      With --base-branch <git ref>, checks for modified cookbooks since <git ref>, default is origin/master
+
+      Examples:
+
+      fhcap cookbook check-bumps
+
+      fhcap cookbook check-bumps --base-branch origin/FH-v3.18
+      LONGDESC
+
+      method_option 'base-branch', :type => :string, :default => 'origin/master', :desc => "Base branch to check modified cookbooks against"
+
+      def check_bumps
+        require "fhcap/tasks/chef/cookbook/check_bumps"
+        Tasks::Chef::Cookbook::CheckBumps.new(task_options(options.dup.merge({modified: true, base_branch: options['base-branch']}))).run
+      end
+
       desc "update_metadata", "Update a cookbooks metadata, defaults to all cookbooks."
 
       shared_options :cookbooks, :modified
@@ -80,6 +103,23 @@ module Fhcap
         Tasks::Chef::Cookbook::Archive.new(task_options(options.dup)).run
       end
 
+      desc "update_attribute", "update_attribute"
+
+      shared_options :cookbooks
+      method_option :'attribute', :type => :string, :desc => "Attribute"
+      method_option :'value', :type => :string, :desc => "Value"
+      method_option :'attribute-file', :type => :string, :desc => "Attribute File (artifacts.rb)"
+
+      def update_attribute
+        require "fhcap/tasks/chef/cookbook/update_attribute"
+        cookbooks = get_cookbooks(options, nil, repo_cookbook_paths('-cookbooks'))
+        say("Cookbook update_attribute for #{cookbooks}", :yellow)
+
+        cookbooks.each do |cookbook|
+          Tasks::Chef::Cookbook::UpdateAttribute.new(task_options(options.dup.merge({cookbook: cookbook}))).run
+        end
+      end
+
     end
   end
-end
+end

data/lib/fhcap/misc.rb

@@ -91,6 +91,13 @@ module Fhcap
         end
       end
 
+      desc "platform_bump VERSION", "Bumps RHMAP Platform Version"
+
+      def platform_bump(version)
+        require "fhcap/tasks/misc/platform_bump"
+        Tasks::Misc::PlatformBump.new(task_options(options.dup.merge({version: version}))).run
+      end
+
     end
   end
-end
+end

data/lib/fhcap/repo.rb

@@ -13,7 +13,7 @@ module Fhcap
 
       shared_options :verbose, :name, :interactive
       method_option 'url', :type => :string, :desc => "Repo url i.e. git@github.com:fheng/fhcap-dev.git"
-      method_option 'clusters-dir', :type => :string, :default => 'clusters', :desc => "Directlory in repo where cluster configs are stored i.e. organisations"
+      method_option 'clusters-dir', :type => :string, :default => 'clusters', :desc => "Directory in repo where cluster configs are stored i.e. organisations"
 
       def add
         require "fhcap/tasks/repo/add"

data/lib/fhcap/repos_helper.rb

@@ -125,6 +125,8 @@ module Fhcap
     def get_cookbooks(options, default=nil, paths=repo_cookbook_paths)
       if options[:modified]
         get_modified_cookbooks(options, paths)
+      elsif options[:dirty]
+        get_dirty_cookbooks(options, paths)
       else
         options[:cookbooks] || (default || cookbook_loader(paths).cookbook_names)
       end
@@ -139,6 +141,15 @@ module Fhcap
       modified
     end
 
+    def get_dirty_cookbooks(options, paths=repo_cookbook_paths)
+      dirty = []
+      cookbook_loader(paths).cookbook_names.each do |name|
+        cookbook_dir = cookbook_loader.cookbooks_by_name[name].root_dir
+        dirty << name if is_dirty?(cookbook_dir)
+      end
+      dirty
+    end
+
     def get_cookbook_meta(name, paths=repo_cookbook_paths)
       cookbook_loader(paths).metadata[name]
     end

data/lib/fhcap/tasks/chef/cookbook/aws/s3_helper.rb

@@ -0,0 +1,21 @@
+require 'aws-sdk'
+require 'fhcap/providers_helper'
+
+module Fhcap
+  module S3Helper
+
+    include Fhcap::ProvidersHelper
+
+    def with_provider_client(provider_name, provider_region)
+      cfg = provider_config(provider_name)
+      region = provider_region || provider_regions(provider_name).keys.first
+      creds = cfg[:credentials]
+      client = Aws::S3::Resource.new(
+          region: region,
+          credentials: Aws::Credentials.new(creds[:'aws-access-key'], creds[:'aws-secret-key'])
+      )
+      yield client
+    end
+
+  end
+end

data/lib/fhcap/tasks/chef/cookbook/check_bumps.rb

@@ -0,0 +1,109 @@
+require 'fhcap/tasks/chef/chef_task_base'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Cookbook
+        class CheckBumps < ChefTaskBase
+
+          attr_reader :base_branch
+
+          def initialize(options)
+            super
+            @base_branch = options[:base_branch]
+          end
+
+          def run
+            cookbooks = get_cookbooks(options, nil, repo_cookbook_paths('-cookbooks'))
+            thor.say("Cookbook Check Bumps for #{cookbooks}", :yellow)
+
+            errors = []
+
+            cookbooks.each do |c|
+              cookbook = cookbook_loader.cookbooks_by_name[c]
+              metadata_file = cookbook.metadata_filenames.first
+
+              if not File.exist?(metadata_file)
+                thor.say("#{cookbook.name} no longer seems to exist", :yellow)
+                next
+              elsif is_new_since?(base_branch, metadata_file)
+                thor.say("#{cookbook.name} seems to be a new cookbook, not checking", :yellow)
+                next
+              end
+
+              version = cookbook.version
+              oldversion = get_old_cookbook_version(base_branch, metadata_file, version)
+
+              if base_branch =~ /master$/
+                min = next_minor_version(oldversion)
+                if minor_bumped_incorrectly?(version, min)
+                  errors << "#{cookbook.name} is at version #{version}, but should be at least #{min} and not be incremented at the patch level"
+                end
+              else
+                min = next_patch_version(oldversion)
+                too_high = next_minor_version(oldversion)
+                if patch_bumped_incorrectly?(version, min, too_high)
+                  errors << "#{cookbook.name} is at version #{version}, but should be at least #{min} and less than #{too_high}"
+                end
+              end
+            end
+
+            unless errors.empty?
+              thor.say("Incorrectly bumped cookbooks:", :red)
+              errors.each { |error|
+                puts error
+              }
+              thor.say("If you haven't changed these cookbooks, then your branch might be behind the base branch (#{base_branch}). In this case, rebasing should make these errors disappear.", :yellow)
+              exit 1
+            end
+          end
+
+          private
+
+          def get_old_cookbook_version(gitref, metadata_file, current_version)
+            Dir.chdir(File.dirname metadata_file) do
+              diff = `git diff #{gitref} -- #{File.basename metadata_file}`
+              oldver = diff.lines.select { |e| e.start_with?("-version") }[0]
+              if oldver
+                oldver.tr("-version '\n", "")
+              else
+                return current_version
+              end
+            end
+          end
+
+          def next_minor_version(version)
+            segments = Gem::Version.new(version).segments
+            segments[1] = segments[1].succ
+            segments[2] = 0
+            segments.join '.'
+          end
+
+          def next_patch_version(version)
+            segments = Gem::Version.new(version).segments
+            segments[2] = segments[2].succ
+            segments.join '.'
+          end
+
+          def is_new_since?(gitref, file)
+            Dir.chdir(File.dirname file) do
+              diff = `git diff --stat --diff-filter=A #{gitref} -- #{File.basename file}`
+              not diff.empty?
+            end
+          end
+
+          def minor_bumped_incorrectly?(version, min)
+            Gem::Version.new(min) > Gem::Version.new(version) or
+              !version.end_with? '.0'
+          end
+
+          def patch_bumped_incorrectly?(version, min, too_high)
+            Gem::Version.new(min) > Gem::Version.new(version) or
+              Gem::Version.new(version) >= Gem::Version.new(too_high)
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/chef/cookbook/list_artifacts.rb

@@ -1,4 +1,6 @@
 require 'fhcap/tasks/chef/chef_task_base'
+require 'fhcap/tasks/chef/cookbook/aws/s3_helper'
+require 'aws-sdk'
 
 module Fhcap
   module Tasks
@@ -6,6 +8,8 @@ module Fhcap
       module Cookbook
         class ListArtifacts < ChefTaskBase
 
+          include Fhcap::S3Helper
+
           def initialize(options)
             super(options)
             @format = options[:format] || 'table'
@@ -13,29 +17,60 @@ module Fhcap
 
           def run
             if print_table?
-              thor.say "Chef::Cookbook::Component::List", :yellow 
+              thor.say "Chef::Cookbook::Component::ListArtifacts", :yellow
             end
+
+            results = print_table? ? [table_header("Component", "Version", "Build", "Artifact", "Available")] : []
+
+            artifact_meta = get_cookbook_meta(['fh-artifact'], 'artifacts')
+            artifact_s3_bucket = artifact_meta['artifact']['s3']['bucket']
+            # Provider must be fheng to have access to S3 artifact buckets
+            s3_provider_name = "aws:fheng"
+            s3_provider_region = "eu-west-1"
+            exit_status = 0
+
             cookbooks = get_cookbooks(options, nil, repo_cookbook_paths('site-cookbooks'))
 
-            results = print_table? ? [table_header("Component", "Version", "Build")] : []
-
-            cookbooks.each do |name|
-              cookbook = cookbook_loader.cookbooks_by_name[name]
-              artifact_attribute_filepath = cookbook.attribute_filenames.find { |e| /artifact.rb/ =~ e }
-              if artifact_attribute_filepath
-                node = ::Chef::Node.new
-                node.from_file(artifact_attribute_filepath)
-                artifact_version = node.default[name]['version']
-                artifact_build = node.default[name]['build']
-                if print_table?
-                  results << table_row(name, artifact_version, artifact_build)
-                else
-                  results << {component: name, version: artifact_version, build: artifact_build}
-                end
+            components_meta = get_cookbook_meta(cookbooks, 'artifact')
+            components_meta.each_attribute do |component_name, component_cfg|
+              component_version = component_cfg['version']
+              component_build = component_cfg['build']
+              component_artifact_filename = component_cfg['artifact_filename'] || "#{component_name}-%s-%s-ubuntu.x64.tar.gz"
+              component_artifact = component_artifact_filename % [component_version, component_build]
+
+              # Trying to handle naming inconsistencies between components and S3 directories
+              case component_name
+              when "unifiedpush", "unifiedpush-auth-server"
+                component_s3_dir = "aerogear-unifiedpush-server"
+              when "fh-metrics"
+                component_s3_dir = "fh-messaging"
+              when "fh-stats"
+                component_s3_dir = "fh-stats"
+                component_artifact = "fh-statsd-%s-%s-ubuntu.x64.tar.gz" % [component_version, component_build]
+              when "migrate"
+                component_s3_dir = "millicore"
+              else
+                component_s3_dir = component_name
+              end
+
+              # Checking that artifact exists
+              artifact_exists = check_artifact_exists(artifact_s3_bucket, s3_provider_name, s3_provider_region, component_s3_dir, component_build, component_artifact)
+
+              # If artifact not found return exit status error
+              unless artifact_exists
+                exit_status = 1
+              end
+
+              if print_table?
+                results << table_row(component_name, component_version, component_build, @component_s3_url, artifact_exists)
+              else
+                results << {component: component_name, version: component_version, build: component_build, artifact: @component_s3_url, available: artifact_exists}
               end
+
             end
             thor.print_table results if print_table?
             thor.say JSON.pretty_generate(results) if print_json?
+            exit exit_status
           end
 
           private
@@ -48,8 +83,29 @@ module Fhcap
             @format == 'json'
           end
 
+          def get_cookbook_meta(cookbooks, attribute_filename)
+            node = ::Chef::Node.new
+            cookbooks.each do |cookbook|
+              cookbook_data = cookbook_loader.cookbooks_by_name[cookbook]
+              cookbook_attribute_filepath = cookbook_data.attribute_filenames.find { |e| /#{attribute_filename}.rb/ =~ e }
+              if cookbook_attribute_filepath
+                node.from_file(cookbook_attribute_filepath)
+              end
+            end
+            node
+          end
+
+          def check_artifact_exists(bucket_name, provider_name, provider_region, component_name, build, artifact_file)
+            with_provider_client(provider_name, provider_region) do |client|
+              bucket = client.bucket(bucket_name)
+              file = "#{component_name}/#{build}/#{artifact_file}"
+              @component_s3_url = bucket.object(file).public_url
+              bucket.object(file).exists?
+            end
+          end
+
         end
       end
     end
   end
-end
+end

data/lib/fhcap/tasks/chef/cookbook/update_attribute.rb

@@ -0,0 +1,51 @@
+require 'fhcap/tasks/chef/chef_task_base'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Cookbook
+        class UpdateAttribute < ChefTaskBase
+
+          attr_reader :name, :cookbook, :attribute, :value, :attribute_file
+
+          def initialize(options)
+            super
+            @cookbook = options[:cookbook]
+            @attribute = options[:attribute]
+            @value = options[:value]
+            @attribute_file = options[:'attribute-file'] || 'default.rb'
+          end
+
+          def run
+            thor.say "Chef::Cookbook::UpdateAttribute cookbook = #{cookbook}, attribute = #{attribute}, value = #{value}", :yellow
+            c = cookbook_loader.cookbooks_by_name[cookbook]
+            unless c
+              exit_with_error("Failed to load cookbook \"#{cookbook}\"!")
+            end
+
+            artifact_attribute_filepath = c.attribute_filenames.find { |e| /#{attribute_file}/ =~ e }
+
+            if artifact_attribute_filepath
+              text = File.read(artifact_attribute_filepath)
+
+              attr = "default"
+              attr_regexp = "default"
+              attribute.to_s.split('.').each do |part|
+                attr = attr + "['#{part}']"
+                attr_regexp = attr_regexp + "\\['#{part}'\\]"
+              end
+
+              replace = text.gsub(/#{attr_regexp} = .+$/, "#{attr} = '#{value}'")
+
+              thor.create_file(artifact_attribute_filepath, replace, {force: !options[:interactive]})
+            else
+              thor.say_status("error", "No attribute file #{attribute_file} found in #{cookbook}", :red)
+            end
+
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/chef/environments/create.rb

@@ -14,6 +14,7 @@ module Fhcap
             @type = options[:type]
             @repo = options[:repo]
             @passwords = {}
+            @efs_enabled = options[:efs_enabled]
           end
 
           def run
@@ -21,7 +22,8 @@ module Fhcap
             env_config = {
                 name: @name,
                 domain: @domain,
-                skip: true
+                skip: true,
+                efs_enabled: @efs_enabled
             }
             create_environment(env_config)
             create_data_bag(env_config)

data/lib/fhcap/tasks/cluster/create_environment.rb

@@ -40,7 +40,8 @@ module Fhcap
               name: "#{name}-#{environment_name}",
               domain: domain,
               type: get_env_type(environment_name),
-              repo: cluster_config[:repo]
+              repo: cluster_config[:repo],
+              efs_enabled: cluster_config[:provider_config][:efs_enabled]
           }
           generate_environment_config
           create_chef_environment(env_cfg)
@@ -160,8 +161,8 @@ module Fhcap
               'mbaas'
             when /ose-single/
               'ose-single'
-            when /oscp/
-              'oscp'
+            when /ocp/
+              'ocp'
             when /single/
               'single'
             when /farm/

data/lib/fhcap/tasks/misc/platform_bump.rb

@@ -0,0 +1,36 @@
+require 'fhcap/tasks/task_base'
+
+module Fhcap
+  module Tasks
+    module Misc
+      class PlatformBump < TaskBase
+
+        attr_reader :version
+
+        def initialize(options)
+          super
+          @version = options[:version]
+        end
+
+        def run
+          thor.say "Misc::PlatformBump version = #{version}", :yellow
+          c = cookbook_loader.cookbooks_by_name['feedhenry_common']
+          unless c
+            exit_with_error("Failed to load cookbook \"#{cookbook}\"!")
+          end
+
+          artifact_attribute_filepath = c.attribute_filenames.find { |e| /platform_info.rb/ =~ e }
+
+          if artifact_attribute_filepath
+            text = File.read(artifact_attribute_filepath)
+            replace = text.gsub(/default\['feedhenry_common'\]\['platform_info'\]\['version'\] = .+$/, "default['feedhenry_common']['platform_info']['version'] = '#{version}'")
+            thor.create_file(artifact_attribute_filepath, replace, {force: !options[:interactive]})
+          else
+            thor.say_status("error", "No artifact file found in feedhenry_common", :red)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/repo/add.rb

@@ -14,10 +14,8 @@ module Fhcap
           @name = options[:name]
 
           @repo_config = config[:repos][@name.to_sym] || {}
-          @repo_config = {
-              url: options[:url],
-              clusters_dir: options[:'clusters-dir'] || 'clusters',
-          }.merge(@repo_config)
+          @repo_config[:url] = options[:url] if options[:url]
+          @repo_config[:clusters_dir] = options[:'clusters-dir'] if options[:'clusters-dir']
 
           @skip_setup = options[:'skip-setup']
         end
@@ -36,7 +34,7 @@ module Fhcap
           config[:repos][name.to_sym] = repo_config
           thor.create_file(config.config_file, JSON.pretty_generate(config.data), :force => true)
           config.reload
-          Checkout.new(@options.dup.merge({:repo => name, :'git-ref' => 'master'})).run
+          Checkout.new(@options.dup.merge({:repo => name})).run
           Tasks::Setup.new(@options).run unless @skip_setup
         end
 
@@ -54,4 +52,4 @@ module Fhcap
       end
     end
   end
-end
+end