fhcap-cli 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65e3070c4f135cab4d5b55260bb771dc576defbc
-  data.tar.gz: 81209d137087b8cdb2053a5b9ec51812666fed32
+  metadata.gz: 3334d314b15975b679b1566f0c11b514c3704f3f
+  data.tar.gz: 2429d1156af144524e6ff176d71127708978298e
 SHA512:
-  metadata.gz: 0590064eade7da691bb7142e5e982009b6fe2e0e425db345f4395e869eba1220e4b719a798ee7f42bcb921be14413e6dcd7cd03045e0ad843827a75adc573ac6
-  data.tar.gz: c881d8f06f8b4d8819bec8b9a4a3bb62ce819dd49631c2528e2ad746209e858371b38cc05e8c66d83038c2c7400bd221a1f7e7bbb4e131170bb7421c0ff9f43a
+  metadata.gz: 3ae8067f89a66214a2df62e822a91685b5d96c7d6e3ac052d62dc2d1c1f732f5db8f5db06053df2af94577d9e495f2cbc35fcde37ef0125f7aa929fccc3b9cde
+  data.tar.gz: b5c3573113861387ac44d4f6963c163b1765be989d15d3ff89c1ecde1ef507fea9160162465695e106f9b7802a33daff6740346ea8b51c201b7517401103d726

data/CHANGELOG.md

@@ -1,4 +1,12 @@
 
+## 0.4.0
+
+* [RHMAP-2732] - Ensure directory exists when finding remote during git checkout of repo
+* [RHMAP-1715] - Add component update tasks.
+* [RHMAP-2535] - Save generated cluster key pairs in the key pairs directory of the new clusters fhcap repo i.e. fhcap-dev/organisations/key_pairs.
+* [RHMAP-2282] - Add support-dmz security group with brno cidrs
+* [RHMAP-2647] - Dynamically configure remote during repo checkout based on configured repos url.
+
 ## 0.3.0
 
 * Implemented cluster create using chef provisioning, supports aws and openstack
@@ -12,4 +20,4 @@
 
 ## 0.1.0
 * Initial release
-* Initial work on setup task
+* Initial work on setup task

data/README.md

@@ -18,6 +18,10 @@ Or install it yourself as:
 
     $ gem install fhcap-cli
 
+The gem itself has quite a few dependencies, to help speed up installation you could skip the docs:
+
+    $ gem install fhcap-cli --no-ri --no-doc
+
 ### Setup
 
 ```
@@ -40,6 +44,9 @@ This creates a new fhcap config, clones required repos and sets up knife based o
 |   `-- fhcap.json
 ```
 
+
+##### repos-dir
+
 If you want to specify a different repos directory, you can do this by passing it in as an option:
 
 ```
@@ -53,6 +60,22 @@ fhcap setup -i
 Local directory where repos will be checked out (/Users/mnairn/src) /Users/mnairn/some_other_repo_dir
 ```
 
+##### fh-src-dir
+
+The default 'fh-src-dir' is set to ~/fhcap/repos, if you already have fh component src checked out or want to use a different directory, you can pass it as an option:
+
+```
+fhcap setup --fh-src-dir /Users/mnairn/src/fheng
+```
+
+or, run setup in interactive mode and change it when prompted:
+
+```
+bundle exec fhcap setup -i
+Local directory where FH component src repos are checked out (/Users/mnairn/src/fheng)
+```
+
+
 #### knife setup (chef server)
 
 After the initial setup, you will need to copy your own chef server credentials into the correct knife directories in order to gain proper access to a chef server.

data/fhcap-cli.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'knife-block', '~> 0.2.1'
   spec.add_dependency 'knife-cookbook-readme', '~> 0.2.1'
   spec.add_dependency 'ruby-progressbar', '~> 1.7.5'
+  spec.add_dependency 'java_properties', '~> 0.0.4'
   spec.add_dependency 'aws-sdk', '~> 2.1.23'
   spec.add_dependency 'chef', '~> 12.4.3'
   spec.add_dependency 'chef-provisioning', '~> 1.4.0'

data/lib/cookbooks/provision/recipes/aws_cluster_create.rb

@@ -4,14 +4,13 @@ include_recipe "provision::aws"
 cluster_config = cluster_config_for(node)
 org_name = cluster_config[:id]
 
+local_key_pairs_dir = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'key_pairs')
 key_pair_name = key_pair_name_for(org_name)
-aws_key_pair key_pair_name
 
-#aws_key_pair 'my-aws-key' do
-#  private_key_path "~boiardi/.ssh/my-aws-key.pem"
-#  public_key_path "~boiardi/.ssh/my-aws-key.pub"
-#  allow_overwrite false # Set to true if you want to regenerate this each chef run
-#end
+aws_key_pair key_pair_name do
+  private_key_path File.join(local_key_pairs_dir, "#{key_pair_name}.pem")
+  public_key_path File.join(local_key_pairs_dir, "#{key_pair_name}.pub")
+end
 
 #Create VPC
 vpc_name = vpc_name_for(org_name)

data/lib/cookbooks/provision/recipes/aws_cluster_create_elb.rb

@@ -20,8 +20,8 @@ cluster_config[:environments].each do |env, env_config|
     lb_certs = {}
     lb_config[:listeners].each_with_index do |listener, index|
       if listener[:server_certificate]
-        certificate_file = File.join(node[:local_repo_path], 'clusters', 'ssl_certs', "#{listener[:server_certificate]}-certificate.pem")
-        private_key_file = File.join(node[:local_repo_path], 'clusters', 'ssl_certs', "#{listener[:server_certificate]}-private-key.pem")
+        certificate_file = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'ssl_certs', "#{listener[:server_certificate]}-certificate.pem")
+        private_key_file = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'ssl_certs', "#{listener[:server_certificate]}-private-key.pem")
 
         if File.exists?(certificate_file) && File.exists?(private_key_file)
           certificate_body = File.open(certificate_file, "rb").read

data/lib/cookbooks/provision/recipes/openstack_cluster_create.rb

@@ -5,7 +5,12 @@ include_recipe "provision::openstack"
 cluster_config = cluster_config_for(node)
 org_name = cluster_config[:id]
 
+local_key_pairs_dir = File.join(node[:local_repo_path], node[:local_repo_clusters_dir], 'key_pairs')
 key_pair_name = key_pair_name_for(org_name)
-fog_key_pair key_pair_name
+
+fog_key_pair key_pair_name do
+  private_key_path File.join(local_key_pairs_dir, "#{key_pair_name}.pem")
+  public_key_path File.join(local_key_pairs_dir, "#{key_pair_name}.pub")
+end
 
 include_recipe "provision::cluster_provision_instances"

data/lib/fhcap/chef-dk/chef_runner.rb

@@ -16,11 +16,13 @@ module Fhcap
 
     attr_reader :cookbook_path
     attr_reader :run_list
+    attr_reader :private_key_paths
     attr_reader :node_attrs
 
-    def initialize(cookbook_path, run_list, node_attrs={})
+    def initialize(cookbook_path, run_list, node_attrs={}, private_key_paths=[])
       @cookbook_path = File.expand_path(cookbook_path)
       @run_list = run_list
+      @private_key_paths = private_key_paths
       @node_attrs = node_attrs
       @formatter = nil
       @ohai = nil
@@ -64,6 +66,7 @@ module Fhcap
       Chef::Config.log_level = :error
       Chef::Config.ssl_verify_mode = :verify_none
       Chef::Config.cookbook_path = cookbook_path
+      Chef::Config.private_key_paths = (Chef::Config.private_key_paths + private_key_paths).compact.uniq
       Chef::Config.color = true
       Chef::Config.diff_disabled = true
 

data/lib/fhcap/cli.rb

@@ -1,5 +1,6 @@
 require 'fhcap/thor_base'
 require 'fhcap/cluster'
+require 'fhcap/component'
 require 'fhcap/cookbook'
 require 'fhcap/repo'
 require 'fhcap/knife'
@@ -17,6 +18,7 @@ module Fhcap
       add_shared_option :interactive, :aliases => "-i", :desc => "Interactive mode (Stops for input)", :type => :boolean, :required => false, :default => false
 
       method_option 'repos-dir', :type => :string, :desc => "Local directory where configured repos will be checked out"
+      method_option 'fh-src-dir', :type => :string, :desc => "Local directory where FH component src repos are checked out"
 
       desc "setup", "Setup the FHCAP CLI"
 
@@ -46,6 +48,9 @@ module Fhcap
       desc "cluster COMMANDS", "Cluster commands"
       subcommand "cluster", Fhcap::CLI::Cluster
 
+      desc "component COMMANDS", "Component commands"
+      subcommand "component", Fhcap::CLI::Component
+
       desc "cookbook COMMANDS", "Cookbook commands"
       subcommand "cookbook", Fhcap::CLI::Cookbook
 

data/lib/fhcap/cluster.rb

@@ -15,7 +15,7 @@ module Fhcap
 
       method_option 'template', :type => :string, :desc => "Cluster template i.e. [1, 3 , 5 node cluster setup, core or mbaas]", :enum => cluster_template_names
       method_option 'git-ref', :type => :string, :desc => "fhcap git ref, can be a branch, tag or commit hash"
-      method_option 'git-remote', :type => :string, :desc => "git remote to use, default is origin"
+      method_option 'git-remote', :type => :string, :desc => "git remote to use, default is the remote for the configured repos url"
       method_option 'domain', :type => :string, :desc => "host names to register with DNS / base host name"
       method_option 'ssl-cert', :type => :string, :desc => "ssl cert to use"
       method_option 'dns-provider-id', :type => :string, :desc => "DNS Provider Id"
@@ -37,7 +37,7 @@ module Fhcap
 
       shared_options :verbose, :name
       method_option 'git-ref', :type => :string, :desc => "fhcap git ref, can be a branch, tag or commit hash"
-      method_option 'git-remote', :type => :string, :desc => "git remote to use, default is origin"
+      method_option 'git-remote', :type => :string, :desc => "git remote to use, default is the remote for the configured repos url"
       method_option 'nodes', :type => :string, :desc => "Query to pass to name"
       method_option 'roles', :type => :string, :desc => "Query to pass to roles"
       method_option 'series', :type => :boolean, :default => false, :desc => "Run provision in series"

data/lib/fhcap/component.rb

@@ -0,0 +1,75 @@
+require 'thor'
+require 'fhcap/thor_base'
+
+module Fhcap
+  module CLI
+    class Component < Fhcap::ThorBase
+
+      add_shared_option :verbose, :aliases => "-v", :desc => "Verbose output", :type => :boolean, :required => false, :default => false
+      add_shared_option :interactive, :aliases => "-i", :desc => "Interactive mode (Stops for input)", :type => :boolean, :required => false, :default => false
+
+      desc "update COMPONENT VERSION BUILD", "Update a single component"
+
+      shared_options :interactive, :verbose
+      method_option 'force', :type => :boolean, :aliases => "-f", :desc => "Will force the component version update even if the artifact cant be found"
+      method_option 'wip', :type => :boolean, :desc => "Work in progress. Limits the meta that is changed during the update. i.e. no readme etc.."
+      method_option 'tag', :type => :string, :desc => "Adds a tag to a components changelog entry"
+
+      def update(component, version, build)
+
+        #1. Update artifact version
+        Fhcap::CLI::Component.new.invoke(:update_artifact, [component, version, build], {interactive: options[:interactive]})
+
+        #2. Update config
+        Fhcap::CLI::Component.new.invoke(:update_artifact, [component], {force: options[:force]})
+
+        #3. Update cookbook
+        clog_entry = "Updated #{component} component version #{version}-#{build}"
+        clog_entry += " (#{options[:tag]})" if options[:tag]
+        Fhcap::CLI::Cookbook.new.invoke(:update, [], {cookbooks: [component], :"changelog-entries" => [clog_entry], wip: options['wip']})
+
+      end
+
+      desc "update_artifact COMPONENT VERSION BUILD", "Update a single component artifact"
+
+      shared_options :interactive
+
+      def update_artifact(component, version, build)
+        require "fhcap/tasks/chef/cookbook/update_artifact"
+        Tasks::Chef::Cookbook::UpdateConfig.new(task_options(options.dup.merge({cookbook: component, version: version, build: build}))).run
+      end
+
+      desc "update_config COMPONENT", "Generates a chef attributes file from the given config file (json or java properties)"
+      long_desc <<-LONGDESC
+      Updates a chef attribute file e.g. site-cookbooks/fh-ngui/attributes/conf.json form a config file stored within the
+      components own git repository e.g. <FH_SRC_DIR>/fh-ngui/config.dev.json. It can take both json config files
+      (all node components) and java properties files (milliore) as input.
+
+      With -f option, the entire contents of the attribute file will be re-generated from the given config file. You probably don't really want to do this!
+
+      With -o <attribute1 attribute2> option, the given array of attributes will be updated from the input config file.
+      e.g To force an update of only the 'logger' attribute, thor config:generate fh-ngui -o logger. For nested attributes,
+      use . notation for attribute names e.g thor config:generate fh-ngui -n -o fhngui.port
+
+      LONGDESC
+      method_option 'overrides', :type => :array, :aliases => "-o", :desc => "Override the given list of values from the config"
+      method_option 'force', :type => :boolean, :aliases => "-f", :default => false, :desc => "Force all attribute values to be overwritten from config file (Re-writes the entire attribute file contents)"
+      method_option 'sort', :type => :boolean, :aliases => "-s", :default => true, :desc => "Sort attribute file attributes alphabetically"
+
+      def update_config(component)
+        require "fhcap/tasks/chef/cookbook/update_config"
+        Tasks::Chef::Cookbook::UpdateConfig.new(task_options(options.dup.merge({cookbook: component}))).run
+      end
+
+      desc "list", "List components and show artifact versions"
+
+      method_option 'format', :type => :string, :default => 'table', :desc => "Format to output data in", :enum => %w{table json}
+
+      def list
+        require "fhcap/tasks/chef/cookbook/list_artifacts"
+        Tasks::Chef::Cookbook::ListArtifacts.new(task_options(options.dup)).run
+      end
+
+    end
+  end
+end

data/lib/fhcap/config.rb

@@ -59,6 +59,10 @@ module Fhcap
       File.join(default_dir, 'tmp')
     end
 
+    def fh_src_dir
+      data[:fh_src_dir] || File.join(default_dir, 'repos')
+    end
+
     def fhcap_cookbook_archive_url
       'https://s3-eu-west-1.amazonaws.com/fhcap/fhcap-cookbooks-0fa68747b476f7ee555a94483877969a30133e76.tgz'
     end
@@ -67,6 +71,7 @@ module Fhcap
     def fhcap_config
       @fhcap_config ||= {
           :repos_dir => File.join(default_dir, 'repos'),
+          :fh_src_dir => File.join(default_dir, 'repos'),
           :repos => {
               :fhcap => {
                   :url => 'git@github.com:fheng/fhcap.git',

data/lib/fhcap/misc/fh_config.rb

@@ -0,0 +1,120 @@
+require 'fhcap/dummy_node'
+
+module Fhcap
+  class FHConfig
+
+    attr_accessor :file, :config, :node
+
+    def initialize
+      #ToDO Should really get rid of this extra config object and just use @node to store everything
+      @config = {}
+      @node = Fhcap::DummyNode.new
+    end
+
+    def add(key, value, force=false, overrides=[])
+      if force || !@config.has_key?(key) || (overrides.include? key.join('.'))
+        @config[key] = parse_value(value)
+      end
+    end
+
+    #Traverse the key and check it isn't overridden at any point further up the chain
+    def has_upper_recipe_override(key)
+      key.pop
+      return true if @config.has_key?(key) && @config[key] =~ /RECIPE_OVERRIDE/
+      has_upper_recipe_override(key) unless key.empty?
+      false
+    end
+
+    def to_chef_attributes(sort=false)
+      attrs = {}
+      if sort
+        conf = {}
+        @config.keys.sort.each do |key|
+          conf[key] = @config[key]
+        end
+      else
+        conf = @config
+      end
+      conf.each do |key, value|
+        #Skip any keys that are being overridden further up the chain
+        unless has_upper_recipe_override(key.clone)
+          #Since we converted all node['blah'] to strings, we need to convert them back
+          #Ahh. this is horrible, must be a better way to do this
+          val = value.to_s
+          val.gsub!(/node(\[['a-zA-Z0-9_-]*\])+/, '#{\&}')
+          if val =~ /^"#\{node(\[['a-zA-Z0-9_-]*\])+\}"$/
+            val.gsub!(/^"#\{|\}"$/, '')
+          end
+
+          val.gsub!(/node.chef_environment/, '#{\&}')
+          if val =~ /^"#\{node.chef_environment\}"$/
+            val.gsub!(/^"#\{|\}"$/, '')
+          end
+
+          if val =~ /^"\{\}"$/
+            val.gsub!(/"/, '')
+          end
+
+          attrs[key.collect { |part| "['#{part}']" }.join()] = val
+        end
+      end
+      attrs
+    end
+
+    def parse_key(key)
+      key
+    end
+
+    def parse_value(value)
+      if value =~ /^[0-9]+$/
+        # Got a number
+      elsif value =~ /^true$|^false$/
+        #Take any quotes off around true/false
+        return value.gsub("'", "")
+      elsif value.is_a? String
+        if is_json?(value)
+          return JSON.parse(value).to_json.inspect
+        elsif value.empty?
+          return "\"\""
+        else
+          value.gsub!(/^"|"$|^'|'$/, "")
+          #value.gsub!(/\\/, "")
+          return "\"#{value}\""
+        end
+      end
+      value
+    end
+
+    def is_json?(value)
+      begin
+        !!JSON.parse(value)
+      rescue
+        false
+      end
+    end
+
+    def from_file(filename)
+    end
+
+    def from_attributes_file(component, filename, force=false)
+      @node.from_file(filename)
+      conf = Fhcap::JsonConfig.new
+      conf.from_json(@node.default_attrs[component][:conf].to_json)
+      conf.config.each() do |k, v|
+        add(k, v, force)
+      end
+    end
+
+    def attribute_file_content(component, sort)
+      content = "# Default conf.rb attributes file, generated by fhcap, do not edit\n\n"
+      @node.included_recipe_attributes.each do |recipe|
+        content += "include_attribute '#{recipe}'\n"
+      end
+      self.to_chef_attributes(sort).collect do |key, value|
+        content += "default['#{component}']['conf']#{key} = #{value}\n"
+      end
+      content
+    end
+
+  end
+end

data/lib/fhcap/misc/java_config.rb

@@ -0,0 +1,36 @@
+require 'fhcap/misc/fh_config'
+require 'java_properties'
+
+module Fhcap
+  class JavaConfig < Fhcap::FHConfig
+
+    def initialize
+      super
+    end
+
+    def from_file(filename, force=false, overrides=[])
+      if File.exists?(filename) && File.readable?(filename)
+        props = JavaProperties::Properties.new(filename)
+        props.each do |key, value|
+          key = key.to_s
+          add(parse_key(key).split("."), value, force, overrides)
+        end
+      else
+        raise IOError, "Cannot open or read #{filename}!"
+      end
+    end
+
+    def parse_key(key)
+      case key
+        when 'feedhenry.domainsuffix'
+          return 'feedhenry.domainsuffix.external'
+        when 'digger.polling'
+          return 'digger.polling.enabled'
+        when 'fh.autosetup'
+          return 'fh.autosetup.enabled'
+        else
+          return key
+      end
+    end
+  end
+end

data/lib/fhcap/misc/json_config.rb

@@ -0,0 +1,59 @@
+require 'fhcap/misc/fh_config'
+require 'json'
+require 'pp'
+
+module Fhcap
+  class JsonConfig < Fhcap::FHConfig
+
+    def initialize
+      super
+    end
+
+    def from_file(filename, force=false, overrides=[])
+      if File.exists?(filename) && File.readable?(filename)
+        json = JSON.parse(IO.read(filename))
+        from_json(json, force, overrides)
+      else
+        raise IOError, "Cannot open or read #{filename}!"
+      end
+    end
+
+    def from_json(json, force=false, overrides=[])
+      keys = get_keys(json)
+      keys.each do |tuple|
+        current = tuple[0]
+        value = tuple[1]
+        add(current, value, force, overrides)
+      end
+    end
+
+    def get_keys(s, c=nil, keys=nil)
+      if s.is_a? String
+        s = JSON.parse(s)
+      end
+      keys ||= []
+      c ||= []
+      s.each_pair do |key, value|
+        c.push(key)
+        if value.class == Hash
+          get_keys(value, c, keys)
+          if value == {}
+            keys.push([c.clone, parse_value(value.to_json)])
+          end
+        elsif value.class == Array
+          #  value.each_with_index do |val, ind|
+          #    c.push(ind.to_s)
+          #    get_keys(val, c, keys)
+          #    c.pop
+          #  end
+          keys.push([c.clone, JSON.parse(value.to_json)])
+        else
+          keys.push([c.clone, parse_value(value.to_json)])
+        end
+        c.pop
+      end
+      keys
+    end
+
+  end
+end

data/lib/fhcap/repo.rb

@@ -33,7 +33,7 @@ module Fhcap
 
       shared_options :verbose, :name
       method_option 'git-ref', :type => :string, :desc => "fhcap git ref, can be a branch, tag or commit hash"
-      method_option 'remote', :type => :string, :default => 'origin', :desc => "Git remote to pull from"
+      method_option 'remote', :type => :string, :desc => "git remote to use, default is the remote for the configured repos url"
 
       def checkout
         require 'fhcap/tasks/repo/checkout'

data/lib/fhcap/tasks/chef/cookbook/list_artifacts.rb

@@ -0,0 +1,53 @@
+require 'fhcap/tasks/chef/chef_task_base'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Cookbook
+        class ListArtifacts < ChefTaskBase
+
+          def initialize(options)
+            super(options)
+            @format = options[:format] || 'table'
+          end
+
+          def run
+            thor.say "Chef::Cookbook::Component::List", :yellow
+            cookbooks = get_cookbooks(options, nil, repo_cookbook_paths('site-cookbooks'))
+
+            results = print_table? ? [table_header("Component", "Version", "Build")] : []
+
+            cookbooks.each do |name|
+              cookbook = cookbook_loader.cookbooks_by_name[name]
+              artifact_attribute_filepath = cookbook.attribute_filenames.find { |e| /artifact.rb/ =~ e }
+              if artifact_attribute_filepath
+                node = ::Chef::Node.new
+                node.from_file(artifact_attribute_filepath)
+                artifact_version = node.default[name]['version']
+                artifact_build = node.default[name]['build']
+                if print_table?
+                  results << table_row(name, artifact_version, artifact_build)
+                else
+                  results << {component: name, version: artifact_version, build: artifact_build}
+                end
+              end
+            end
+            thor.print_table results if print_table?
+            thor.say JSON.pretty_generate(results) if print_json?
+          end
+
+          private
+
+          def print_table?
+            @format == 'table'
+          end
+
+          def print_json?
+            @format == 'json'
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/chef/cookbook/update_artifact.rb

@@ -0,0 +1,39 @@
+require 'fhcap/tasks/chef/chef_task_base'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Cookbook
+        class UpdateArtifact < ChefTaskBase
+
+          attr_reader :name, :version, :build
+
+          def initialize(options)
+            super
+            @name = options[:cookbook]
+            @version = options[:version]
+            @build = options[:build]
+          end
+
+          def run
+            thor.say "Chef::Cookbook::UpdateArtifact cookbook = #{name}, version = #{version}, build = #{build}", :yellow
+            cookbook = cookbook_loader.cookbooks_by_name[name]
+
+            artifact_attribute_filepath = cookbook.attribute_filenames.find { |e| /artifact.rb/ =~ e }
+
+            if artifact_attribute_filepath
+              text = File.read(artifact_attribute_filepath)
+              replace = text.gsub(/default\['#{name}'\]\['version'\] = .+$/, "default['#{name}']['version'] = '#{version}'")
+              replace = replace.gsub(/default\['#{name}'\]\['build'\] = .+$/, "default['#{name}']['build'] = '#{build}'")
+              thor.create_file(artifact_attribute_filepath, replace, {force: !options[:interactive]})
+            else
+              thor.say_status("error", "No artifact file found for #{name}", :red)
+            end
+
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/chef/cookbook/update_config.rb

@@ -0,0 +1,78 @@
+require 'fhcap/tasks/chef/chef_task_base'
+require 'fhcap/misc/java_config'
+require 'fhcap/misc/json_config'
+
+module Fhcap
+  module Tasks
+    module Chef
+      module Cookbook
+        class UpdateConfig < ChefTaskBase
+
+          COMP_CFG_MAP = {
+              "fh-aaa" => "fh-aaa/config/dev.json",
+              "millicore" => "millicore/src/main/resources/config/cluster-config.properties",
+              "fh-reaper" => "millicore/src/main/resources/config/reaper.properties",
+              "fh-supercore" => "fh-supercore/config/dev.json",
+              "fh-docs" => ["fh-docs/config/development.json", "fh-docs/config/config.json"],
+              "fh-doxy" => "fh-doxy/config/dev.json",
+              "fh-messaging" => "fh-messaging/fh-messaging/config/conf.json",
+              "fh-metrics" => "fh-messaging/fh-metrics/config/conf.json",
+              "fh-ditch" => "fh-ditch/config/dev.json",
+              "fh-scm" => "fh-scm/config/dev.json",
+              "fh-stats" => "fh-stats/fh-statsd/config/dev.json",
+              "fh-dynoman" => "fh-dynoman/config/dev.json",
+              "fh-proxy" => "fh-proxy/config/dev.json",
+              "fh-ngui" => "fh-ngui/config/dev.json",
+              "fh-appstore" => "fh-appstore/config/dev.json",
+              "fh-digger" => "fh-digger/config/dev.json",
+              "fh-mbaas" => "fh-mbaas/config/dev.json",
+          }
+
+          attr_reader :name, :overrides, :sort, :force
+
+          def initialize(options)
+            super
+            @name = options[:cookbook]
+            @overrides = options[:overrides] || []
+            @sort = options[:sort] || true
+            @force = options[:force] || false
+          end
+
+          def run
+            thor.say "Chef::Cookbook::UpdateConfig cookbook = #{name}", :yellow
+            cookbook = cookbook_loader.cookbooks_by_name[name]
+
+            conf_attr_file = cookbook.attribute_filenames.find { |e| /conf.rb/ =~ e }
+            exit_with_error("No cookbook conf.rb attribute file found for #{name}") unless conf_attr_file
+
+            conf_src_files = cfg_lookup(name)
+            exit_with_error("No local src config file found for #{name}") if conf_src_files.empty?
+
+            thor.say "conf_attr_file = #{conf_attr_file}"
+            thor.say "conf_src_files = #{conf_src_files}"
+
+            conf_src_files.each() do |file|
+              if File.extname(file) =~ /.properties/
+                cfg = Fhcap::JavaConfig.new
+              else
+                cfg = Fhcap::JsonConfig.new
+              end
+              cfg.from_attributes_file(name, conf_attr_file)
+              cfg.from_file(file, force, overrides)
+              thor.create_file(conf_attr_file, cfg.attribute_file_content(name, sort))
+            end
+
+          end
+
+          private
+
+          def cfg_lookup(component)
+            cfg_paths = [COMP_CFG_MAP[component]].flatten.compact
+            cfg_paths.map! { |path| File.join(config.fh_src_dir, path) }
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/chef/provisioning/chef_provisioning_task_base.rb

@@ -17,6 +17,7 @@ module Fhcap
               @cluster_config = JSON.parse(IO.read(@cluster_filepath), {:symbolize_names => true})
               @cluster_config.merge!({
                                          local_repo_path: repo_dir(@cluster_config[:repo]),
+                                         local_repo_clusters_dir: repo_clusters_dir(@cluster_config[:repo]),
                                          chef_server_config: chef_server_config_hash_for(@cluster_config[:chef_server]),
                                          provider_credentials:  provider_credentials(@cluster_config[:provider_id])
                                      })
@@ -28,7 +29,8 @@ module Fhcap
 
           def do_chef_run(run_list)
             cookbook_path = File.join(Fhcap.source_root, 'lib', 'cookbooks')
-            Fhcap::ChefRunner.new(cookbook_path, run_list, @cluster_config).converge
+            private_key_paths = [File.join(cluster_config[:local_repo_path], cluster_config[:local_repo_clusters_dir], 'key_pairs')]
+            Fhcap::ChefRunner.new(cookbook_path, run_list, cluster_config, private_key_paths).converge
           end
 
         end

data/lib/fhcap/tasks/cluster/create.rb

@@ -110,7 +110,7 @@ module Fhcap
           ask_config(required_config, cluster_config)
           cluster_config[:driver] = provider_type(cluster_config[:provider_id])
           send(:"generate_cluster_config_#{cluster_config[:driver]}")
-          driver_template_file = File.join('templates', 'cluster', cluster_config[:driver], "common.json.erb")
+          driver_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', cluster_config[:driver], "common.json.erb")
           driver_template_config = template_as_object(driver_template_file, cluster_config)
           cluster_config.merge!(driver_template_config)
         end
@@ -131,11 +131,11 @@ module Fhcap
         end
 
         def create_cluster_environments
-          env_template_file = File.join('templates', 'cluster', "#{cluster_config[:template]}.json.erb")
+          env_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', "#{cluster_config[:template]}.json.erb")
           template_config = template_as_object(env_template_file, cluster_config)
 
           template_config[:environments].each do |env|
-            env_template_file = File.join('templates', 'cluster', cluster_config[:driver], "#{env[:template]}.json.erb")
+            env_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', cluster_config[:driver], "#{env[:template]}.json.erb")
             template_config = template_as_object(env_template_file, cluster_config)
 
             env_domain = [env[:'domain-prefix'], cluster_config[:domain]].compact.join('.')

data/lib/fhcap/tasks/repo/checkout.rb

@@ -14,7 +14,7 @@ module Fhcap
           @gitref = options[:"git-ref"]
           @repo = options[:repo] || options[:name]
           @fallback_gitref = options[:'fallback-git-ref']
-          @remote = options[:remote] || options[:'git-remote'] || 'origin'
+          @remote = options[:remote] || options[:'git-remote'] || find_remote(@repo)
         end
 
         def run
@@ -41,6 +41,30 @@ module Fhcap
 
         private
 
+        def find_remote(repo)
+          remote = 'origin'
+          dir = repo_dir(repo)
+          if Dir.exists? dir
+            Dir.chdir repo_dir(repo) do
+              remotes = `git remote`.split
+              if remotes.length > 1
+                url = repo_cfg(repo)[:url]
+                remotes.each do |r|
+                  remote_url = `git config --get remote.#{r}.url`.strip
+                  if remote_url == url
+                    remote = r
+                    break
+                  end
+                end
+                remote
+              else
+                remotes.first
+              end
+            end
+          end
+          remote
+        end
+
         def git_clone(repo)
           Dir.exists?(repo_dir(repo)) ? true : do_clone_cmd(repo)
         end

data/lib/fhcap/tasks/setup.rb

@@ -12,6 +12,7 @@ module Fhcap
 
         opts = {
             repos_dir: options[:'repos-dir'],
+            fh_src_dir: options[:'fh-src-dir'],
         }.delete_if { |k, v| v.nil? || v.empty? }
 
         @fhcap_config.merge!(opts)
@@ -31,6 +32,10 @@ module Fhcap
             repos_dir: {
                 msg: "Local directory where repos will be checked out",
                 options: {:path => true}
+            },
+            fh_src_dir: {
+                msg: "Local directory where FH component src repos are checked out",
+                options: {:path => true}
             }
         }
       end

data/lib/fhcap/version.rb

@@ -1,3 +1,3 @@
 module Fhcap
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/spec/fhcap/tasks/chef/cookbook/list_artifacts_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+require "fhcap/tasks/chef/cookbook/list_artifacts"
+
+describe Fhcap::Tasks::Chef::Cookbook::ListArtifacts do
+
+  subject {
+    Fhcap::Tasks::Chef::Cookbook::ListArtifacts.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    {}
+  end
+
+  context "table" do
+
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :format => 'table'
+      }
+    end
+
+    describe "#initialize" do
+      specify { expect(subject.instance_variable_get(:@format)).to eq('table') }
+    end
+
+  end
+
+  context "json" do
+
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :format => 'json'
+      }
+    end
+
+    describe "#initialize" do
+      specify { expect(subject.instance_variable_get(:@format)).to eq('json') }
+    end
+
+  end
+
+
+end

data/spec/fhcap/tasks/chef/cookbook/update_artifact_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require "fhcap/tasks/chef/cookbook/update_artifact"
+
+describe Fhcap::Tasks::Chef::Cookbook::UpdateArtifact do
+
+  subject {
+    Fhcap::Tasks::Chef::Cookbook::UpdateArtifact.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    {}
+  end
+
+  let(:options) do
+    {
+        :config => config,
+        :thor => thor,
+        :cookbook => 'testname',
+        :version => '9.9.9',
+        :build => '999'
+    }
+  end
+
+  describe "#initialize" do
+    specify { expect(subject.name).to eq('testname') }
+    specify { expect(subject.version).to eq('9.9.9') }
+    specify { expect(subject.build).to eq('999') }
+  end
+
+end

data/spec/fhcap/tasks/chef/cookbook/update_config_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+require "fhcap/tasks/chef/cookbook/update_config"
+
+describe Fhcap::Tasks::Chef::Cookbook::UpdateConfig do
+
+  subject {
+    Fhcap::Tasks::Chef::Cookbook::UpdateConfig.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    {}
+  end
+
+  context 'default' do
+
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :cookbook => 'testname'
+      }
+    end
+
+    describe "#initialize" do
+      specify { expect(subject.name).to eq('testname') }
+      specify { expect(subject.overrides).to eq([]) }
+      specify { expect(subject.sort).to eq(true) }
+      specify { expect(subject.force).to eq(false) }
+    end
+
+  end
+
+  context 'force' do
+
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :cookbook => 'testname',
+          :force => true
+      }
+    end
+
+    describe "#initialize" do
+      specify { expect(subject.name).to eq('testname') }
+      specify { expect(subject.overrides).to eq([]) }
+      specify { expect(subject.sort).to eq(true) }
+      specify { expect(subject.force).to eq(true) }
+    end
+
+  end
+
+  context 'overrides' do
+
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :cookbook => 'testname',
+          :overrides => ['some.override']
+      }
+    end
+
+    describe "#initialize" do
+      specify { expect(subject.name).to eq('testname') }
+      specify { expect(subject.overrides).to eq(['some.override']) }
+      specify { expect(subject.sort).to eq(true) }
+      specify { expect(subject.force).to eq(false) }
+    end
+
+  end
+
+end

data/templates/cluster/aws/common.json.erb

@@ -38,6 +38,16 @@
           "sources": ["83.147.149.210/32", "46.38.161.225/32", "54.229.76.48/32", "79.125.117.182/32"]
         }
       ]
+    },
+    "support-dmz": {
+      "authorize_ingress": [
+        {
+          "protocols": ["tcp"],
+          "start": 8811,
+          "end": 8811,
+          "sources": ["213.175.37.8/29", "149.11.118.8/29"]
+        }
+      ]
     }
   }
 }

data/templates/cluster/aws/core-3node.json.erb

@@ -2,15 +2,15 @@
   "domain": "<%= config[:domain] %>",
   "subnets": {
     "1a": {
-      "cidr": "64/26",
+      "cidr": "0/26",
       "availability_zone": "1a"
     },
     "1b": {
-      "cidr": "128/26",
+      "cidr": "64/26",
       "availability_zone": "1b"
     },
     "1c": {
-      "cidr": "192/26",
+      "cidr": "128/26",
       "availability_zone": "1c"
     }
   },

data/templates/cluster/aws/core-small-9node.json.erb

@@ -2,15 +2,15 @@
   "domain": "<%= config[:domain] %>",
   "subnets": {
     "1a": {
-      "cidr": "64/26",
+      "cidr": "0/26",
       "availability_zone": "1a"
     },
     "1b": {
-      "cidr": "128/26",
+      "cidr": "64/26",
       "availability_zone": "1b"
     },
     "1c": {
-      "cidr": "192/26",
+      "cidr": "128/26",
       "availability_zone": "1c"
     }
   },

data/templates/cluster/aws/mbaas-3node.json.erb

@@ -2,15 +2,15 @@
   "domain": "<%= config[:domain] %>",
   "subnets": {
     "1a": {
-      "cidr": "64/26",
+      "cidr": "0/26",
       "availability_zone": "1a"
     },
     "1b": {
-      "cidr": "128/26",
+      "cidr": "64/26",
       "availability_zone": "1b"
     },
     "1c": {
-      "cidr": "192/26",
+      "cidr": "128/26",
       "availability_zone": "1c"
     }
   },

data/templates/cluster/aws/nginx-test.json.erb

@@ -2,11 +2,11 @@
   "domain": "<%= config[:domain] %>",
   "subnets": {
     "1a": {
-      "cidr": "64/26",
+      "cidr": "0/26",
       "availability_zone": "1a"
     },
     "1b": {
-      "cidr": "128/26",
+      "cidr": "64/26",
       "availability_zone": "1b"
     }
   },

data/templates/cluster/aws/single.json.erb

@@ -2,15 +2,15 @@
   "domain": "<%= config[:domain] %>",
   "subnets": {
     "1a": {
-      "cidr": "64/26",
+      "cidr": "0/26",
       "availability_zone": "1a"
     },
     "1b": {
-      "cidr": "128/26",
+      "cidr": "64/26",
       "availability_zone": "1b"
     },
     "1c": {
-      "cidr": "192/26",
+      "cidr": "128/26",
       "availability_zone": "1c"
     }
   },
@@ -82,7 +82,21 @@
         "subnet": "1a"
       },
       "load_balancers": ["studio"],
-      "run_list": ["role[fhcap]"]
+      "run_list": [
+        "role[mysql_server]",
+        "role[mongo_server]",
+        "role[platform_mongo_server]",
+        "role[gitlab-shell_server]",
+        "role[rabbitmq_server]",
+        "role[app_server]",
+        "role[pooled_server]",
+        "role[dyno_server]",
+        "role[fh-reaper]",
+        "role[fh-stats]",
+        "role[management_server]",
+        "recipe[tmuxinator]",
+        "recipe[feedhenry_common::fhctl]"
+      ]
     }
   }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fhcap-cli
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Michael Nairn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-20 00:00:00.000000000 Z
+date: 2015-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -122,6 +122,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.7.5
+- !ruby/object:Gem::Dependency
+  name: java_properties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -363,6 +377,7 @@ files:
 - lib/fhcap/chef-dk/chef_runner.rb
 - lib/fhcap/cli.rb
 - lib/fhcap/cluster.rb
+- lib/fhcap/component.rb
 - lib/fhcap/config.rb
 - lib/fhcap/cookbook.rb
 - lib/fhcap/dummy_node.rb
@@ -371,13 +386,19 @@ files:
 - lib/fhcap/knife.rb
 - lib/fhcap/knife_helper.rb
 - lib/fhcap/misc.rb
+- lib/fhcap/misc/fh_config.rb
+- lib/fhcap/misc/java_config.rb
+- lib/fhcap/misc/json_config.rb
 - lib/fhcap/provider.rb
 - lib/fhcap/providers_helper.rb
 - lib/fhcap/repo.rb
 - lib/fhcap/repos_helper.rb
 - lib/fhcap/tasks/chef/chef_task_base.rb
 - lib/fhcap/tasks/chef/cookbook/list.rb
+- lib/fhcap/tasks/chef/cookbook/list_artifacts.rb
+- lib/fhcap/tasks/chef/cookbook/update_artifact.rb
 - lib/fhcap/tasks/chef/cookbook/update_changelog.rb
+- lib/fhcap/tasks/chef/cookbook/update_config.rb
 - lib/fhcap/tasks/chef/cookbook/update_metadata.rb
 - lib/fhcap/tasks/chef/cookbook/update_readme.rb
 - lib/fhcap/tasks/chef/cookbook/update_version.rb
@@ -421,6 +442,9 @@ files:
 - lib/fhcap/thor_base.rb
 - lib/fhcap/version.rb
 - spec/fhcap/cli_spec.rb
+- spec/fhcap/tasks/chef/cookbook/list_artifacts_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_artifact_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_config_spec.rb
 - spec/fhcap/tasks/cluster/create_spec.rb
 - spec/fhcap/tasks/knife/add_spec.rb
 - spec/fhcap/tasks/knife/remove_spec.rb
@@ -492,6 +516,9 @@ specification_version: 4
 summary: FHCAP Command Line Tool
 test_files:
 - spec/fhcap/cli_spec.rb
+- spec/fhcap/tasks/chef/cookbook/list_artifacts_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_artifact_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_config_spec.rb
 - spec/fhcap/tasks/cluster/create_spec.rb
 - spec/fhcap/tasks/knife/add_spec.rb
 - spec/fhcap/tasks/knife/remove_spec.rb