fhcap-cli 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4068e4a42ccdb6ecf9ed5f1e74e34f8a7d1050ea
-  data.tar.gz: 9e703a039cda7f9572dbb78254ccc24d36b9a495
+  metadata.gz: c70a588e2af142866f7769a2f1b8ee1a8cf669bf
+  data.tar.gz: 12ee618fb29f5376cddd76ccccd9334cb4315f58
 SHA512:
-  metadata.gz: c7c1c547d42be6f597042723e9f00e3a46cc30e93750bc93b65e50d2bc08436f5b1a1a2339a5be8d46e5a2eb97067e79aa89b85fab72f88fb6d81a1d23b2a883
-  data.tar.gz: 95daf1cb3fa9e7dc7f58b5574daf68a4084fff9cef39706f8f2ab3e89a9e3ee8c9124a8a4f27e505cea9ae6d5c15b4e994c2b83eead184a1e824d885d5e641d4
+  metadata.gz: ae42d497ed5c59525b800a31f8a09d5a34c6a00a9eb67989287e38da4e3a06075360aec7161f6e063b7a48ea8d317996c1b568665815fcf894e5f513e2a49d20
+  data.tar.gz: d769c3a259218b4314b0472bf1d83aca477cb4ef37d516698b3389a022fad0759ea37ced456d6949b6522cd58ff8f5481f05aa2f62d21383c39ddbb03d0d6f2e

data/CHANGELOG.md

@@ -1,4 +1,14 @@
 
+## 0.4.4
+
+* [RHMAP-3188] - Multi node template improvements. Breaks out clustered component (galera, rabbitmq and mongo) provisions into separate creation steps allowing them to be handled in isolation from the full chef run.
+* [RHMAP-3473] - Add 3 node and single farm template (Android only) for aws and openstack.
+* Generate Random api key for messaging and metrics mbaas
+* [RHMAP-3278] - Add erubis as a dependency in gemspec.
+* [RHMAP-2928] - Update fhcap dev to point at QED1 for Android builds
+* Use dev.json for fh-metrics & fh-messaging
+* [RHMAP-3667] - Show examples when prompting for input during cluster create
+
 ## 0.4.3
 
 * [RHMAP-3056] - Adds the option to use 'local' chef server, removes the need to have access to a public server to provision anything.

data/fhcap-cli.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'thor', '= 0.19.1'
   spec.add_dependency 'sshkey', '= 1.6.1'
   spec.add_dependency 'public_suffix', '= 1.5.2'
+  spec.add_dependency 'erubis', '= 2.7.0'
   spec.add_dependency 'launchy', '= 2.4.3'
   spec.add_dependency 'librarian-chef', '= 0.0.4'
   spec.add_dependency 'knife-block', '= 0.2.1'

data/lib/cookbooks/provision/recipes/cluster_create_instances.rb

@@ -1,11 +1,12 @@
 with_cluster_instances(node) do |cluster_instances|
   cluster_instances.each do |chef_environment, instances|
     with_chef_environment chef_environment do
-      machine_batch do
+      machine_batch 'cluster_create_instances' do
         instances.each do |name, cfg|
           machine name do
             chef_environment chef_environment
             machine_options(cfg[:machine_options])
+            run_list []
           end
         end
       end

data/lib/cookbooks/provision/recipes/cluster_provision_instances.rb

@@ -1,12 +1,13 @@
 with_cluster_instances(node) do |cluster_instances|
   cluster_instances.each do |chef_environment, instances|
     with_chef_environment chef_environment do
-      machine_batch do
+      machine_batch 'cluster_provision_instances' do
         instances.each do |name, cfg|
           machine name do
             chef_environment chef_environment
             run_list cfg[:instance_config][:run_list]
             machine_options(cfg[:machine_options])
+            converge true
           end
         end
       end

data/lib/cookbooks/provision/recipes/galera_reset_cluster.rb

@@ -0,0 +1,62 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+# Help on verifying a galera cluster can be found here http://galeracluster.com/documentation-webpages/testingcluster.html
+
+with_cluster_instances(node) do |cluster_instances|
+  cluster_instances.each do |chef_environment, instances|
+    with_chef_environment chef_environment do
+
+      search_role = 'galera.*_server'
+
+      search_nodes = search(:node, "chef_environment:#{chef_environment} AND roles:#{search_role}").map { |n| n.name }
+
+      run_list_query = /role\[#{search_role}\]/
+
+      galera_instances = instances.select do |name, cfg|
+        if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
+          tmp_run_list = []
+          tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
+          cfg[:instance_config][:tmp_run_list] = tmp_run_list
+          true
+        else
+          false
+        end
+      end
+
+      #Delete the installed flag to force a re-install of galera, Note DO NOT run this in a prod setup that is being used for real.
+      galera_instances.each do |name, cfg|
+        machine_execute "galera_reset_cluster - #{name}: Remove /root/.galera_installed" do
+          command 'sudo rm -rf /root/.galera_installed'
+          machine name
+        end
+      end
+
+      # Chef run to force the install recipe for galera to run, depending on weather nodes exist on the server already or not, we will need to do this once or twice.
+      batch_provisions = search_nodes.length > 1 ? 1 : 2
+      batch_provisions.times do |index|
+        machine_batch "galera_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
+          galera_instances.each do |name, cfg|
+            machine name do
+              chef_environment chef_environment
+              machine_options(cfg[:machine_options])
+              run_list cfg[:instance_config][:tmp_run_list]
+              converge true
+            end
+          end
+        end
+      end
+
+      # Chef run on each node in series to setup clustering
+      galera_instances.each do |name, cfg|
+        machine name do
+          chef_environment chef_environment
+          machine_options(cfg[:machine_options])
+          run_list cfg[:instance_config][:tmp_run_list]
+          converge true
+        end
+      end
+
+    end
+  end
+end

data/lib/cookbooks/provision/recipes/mongo_reset_cluster.rb

@@ -0,0 +1,45 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+with_cluster_instances(node) do |cluster_instances|
+  cluster_instances.each do |chef_environment, instances|
+    with_chef_environment chef_environment do
+
+      search_role = '.*mongo.*_server'
+
+      run_list_query = /role\[#{search_role}\]/
+
+      mongo_instances = instances.select do |name, cfg|
+        if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
+          tmp_run_list = []
+          tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
+          cfg[:instance_config][:tmp_run_list] = tmp_run_list
+          true
+        else
+          false
+        end
+      end
+
+      # It always take 3 provisions for mongo to be working in a clustered setup.
+      #
+      # 1. install mongo and update chef server
+      # 2. configure mongo configs with all the other nodes and setup replicaset
+      # 3. Add any users to what should be a now working mongo cluster
+
+      batch_provisions = 3
+      batch_provisions.times do |index|
+        machine_batch "mongo_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
+          mongo_instances.each do |name, cfg|
+            machine name do
+              chef_environment chef_environment
+              machine_options(cfg[:machine_options])
+              run_list cfg[:instance_config][:tmp_run_list]
+              converge true
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/cookbooks/provision/recipes/post_create_instances.rb

@@ -0,0 +1,4 @@
+
+include_recipe "provision::galera_reset_cluster"
+include_recipe "provision::rabbitmq_reset_cluster"
+include_recipe "provision::mongo_reset_cluster"

data/lib/cookbooks/provision/recipes/rabbitmq_reset_cluster.rb

@@ -0,0 +1,71 @@
+include_recipe 'provision::common'
+include_recipe "provision::#{node['driver']}"
+
+# Help on verifying a rabbitmq cluster can be found here https://www.rabbitmq.com/clustering.html
+
+with_cluster_instances(node) do |cluster_instances|
+  cluster_instances.each do |chef_environment, instances|
+    with_chef_environment chef_environment do
+
+      search_role = 'rabbitmq_server'
+
+      run_list_query = /role\[#{search_role}\]/
+
+      rabbitmq_instances = instances.select do |name, cfg|
+        if cfg[:instance_config] && cfg[:instance_config][:run_list] && cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }
+          tmp_run_list = []
+          tmp_run_list << cfg[:instance_config][:run_list][cfg[:instance_config][:run_list].index { |s| s =~ run_list_query }]
+          cfg[:instance_config][:tmp_run_list] = tmp_run_list
+          true
+        else
+          false
+        end
+      end
+
+      # It always take 2 provisions for rabbitmq to be working in a clustered setup
+      #
+      # 1. install rabbit and update chef server
+      # 2. configure rabbit configs with all the other nodes and setup cluster
+
+      batch_provisions = 2
+      batch_provisions.times do |index|
+        machine_batch "rabbitmq_reset_cluster: Batch Provison #{index + 1}/#{batch_provisions}" do
+          rabbitmq_instances.each do |name, cfg|
+            machine name do
+              chef_environment chef_environment
+              machine_options(cfg[:machine_options])
+              run_list cfg[:instance_config][:tmp_run_list]
+              converge true
+            end
+          end
+        end
+      end
+
+      rabbitmq_instances.each do |name, cfg|
+        machine_execute 'sudo rabbitmqctl stop_app' do
+          machine name
+        end
+
+        machine_execute 'sudo rabbitmqctl reset' do
+          machine name
+        end
+
+        machine_execute 'sudo rabbitmqctl start_app' do
+          machine name
+        end
+      end
+
+      machine_batch "rabbitmq_reset_cluster: Batch Provison 1/1" do
+        rabbitmq_instances.each do |name, cfg|
+          machine name do
+            chef_environment chef_environment
+            machine_options(cfg[:machine_options])
+            run_list cfg[:instance_config][:tmp_run_list]
+            converge true
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/cookbooks/provision/recipes/restart_services.rb

@@ -1,24 +1,18 @@
 include_recipe 'provision::common'
 include_recipe "provision::#{node['driver']}"
 
-cluster_config = cluster_config_for(node)
-org_name = cluster_config[:id]
+with_cluster_instances(node) do |cluster_instances|
+  cluster_instances.each do |chef_environment, instances|
+    with_chef_environment chef_environment do
 
-cluster_config[:environments].each do |env, env_config|
-  chef_environment = fh_name_for(org_name, env)
+      instances.each do |name, cfg|
 
-  nodes = search(:node, "chef_environment:#{chef_environment}")
-
-  with_chef_environment chef_environment do
-
-    nodes.each do |n|
-      machine_execute 'sudo fhctl restart' do
-        live_stream true
-        machine n.name
-        only_if {n.recipes.include? 'feedhenry_common::fhctl'}
+        machine_execute 'sudo fhctl restart' do
+          live_stream true
+          machine name
+        end
       end
-    end
 
+    end
   end
-
 end

data/lib/cookbooks/provision/recipes/seed_cookbooks.rb

@@ -16,19 +16,20 @@ cluster_config[:environments].each do |env, env_config|
   with_chef_environment chef_environment do
 
     nodes.each do |n|
-      machine_execute "Seed #{target_dir}/cookbooks" do
+      machine_execute "#{n.name} seed #{target_dir}/cookbooks" do
         command "sudo tar -xvzf #{local_archive} -C #{target_dir} --keep-old-files"
         live_stream false
         machine n.name
         action :nothing
       end
 
-      machine_file local_archive do
+      machine_file "#{n.name} #{local_archive}" do
+        path local_archive
         machine  n.name
         local_path node[:local_cookbook_archive]
         action :upload
         only_if { node[:local_cookbook_archive]}
-        notifies :run, "machine_execute[Seed #{target_dir}/cookbooks]", :immediately
+        notifies :run, "machine_execute[#{n.name} seed #{target_dir}/cookbooks]", :immediately
       end
 
     end

data/lib/fhcap/cluster.rb

@@ -113,6 +113,10 @@ module Fhcap
 
       method_option 'name', :type => :string, :required => true, :desc => 'Cluster name'
       method_option 'run-list', :type => :array, :required => true, :desc => 'Chef Provisioning Run List i.e provision::cluster_provision'
+      chef_provision_steps = %w{server-bootstrap}
+      chef_provision_steps.each do |step|
+        method_option :"skip-#{step}", :type => :boolean, :default => false, :desc => "Skip #{step}"
+      end
 
       def chef_provision
         require 'fhcap/tasks/cluster/chef_provisioning_task'

data/lib/fhcap/tasks/chef/chef_server_task.rb

@@ -56,12 +56,15 @@ module Fhcap
           local_repo = repo_dir(@chef_repo)
           thor.say "[Chef Zero Server] - Syncing nodes from #{local_repo}"
           knife_upload(local_repo, @chef_server, ['/nodes'])
+          knife_upload(local_repo, @chef_server, ['/data_bags/aws_*'])
           begin
             block.call
           ensure
             thor.say "[Chef Zero Server] - Syncing nodes to #{local_repo}"
             FileUtils.rm_rf(File.join(local_repo, 'nodes'))
+            FileUtils.rm_rf Dir.glob(File.join(local_repo, 'data_bags/aws_*'))
             knife_download(local_repo, @chef_server, ['/nodes'])
+            knife_download(local_repo, @chef_server, ['/data_bags/aws_*'])
           end
         end
 

data/lib/fhcap/tasks/chef/cookbook/update_config.rb

@@ -15,8 +15,8 @@ module Fhcap
               "fh-supercore" => "fh-supercore/config/dev.json",
               "fh-docs" => ["fh-docs/config/development.json", "fh-docs/config/config.json"],
               "fh-doxy" => "fh-doxy/config/dev.json",
-              "fh-messaging" => "fh-messaging/fh-messaging/config/conf.json",
-              "fh-metrics" => "fh-messaging/fh-metrics/config/conf.json",
+              "fh-messaging" => "fh-messaging/fh-messaging/config/dev.json",
+              "fh-metrics" => "fh-messaging/fh-metrics/config/dev.json",
               "fh-ditch" => "fh-ditch/config/dev.json",
               "fh-scm" => "fh-scm/config/dev.json",
               "fh-stats" => "fh-stats/fh-statsd/config/dev.json",

data/lib/fhcap/tasks/chef/cookbook/update_metadata.rb

@@ -1,5 +1,6 @@
 require 'fhcap/tasks/chef/chef_task_base'
 require 'fhcap/dummy_node'
+require 'erubis'
 
 module Fhcap
   module Tasks

data/lib/fhcap/tasks/chef/provisioning/create.rb

@@ -15,9 +15,8 @@ module Fhcap
 
               unless options[:'skip-provision']
                 seed_cookbooks if local_chef_server?
+                do_chef_run("provision::post_create_instances")
                 do_chef_run("provision::cluster_provision")
-                do_chef_run("provision::reset_rabbitmq")
-                do_chef_run("provision::restart_services")
               end
             end
           end

data/lib/fhcap/tasks/chef/provisioning/provision.rb

@@ -9,7 +9,7 @@ module Fhcap
           def run
             thor.say "Chef::Provisioning::Provision #{@name}", :yellow
             with_chef_server do
-              seed_cookbooks if local_chef_server?
+              #seed_cookbooks if local_chef_server?
               do_chef_run("provision::cluster_provision")
             end
           end

data/lib/fhcap/tasks/cluster/chef_provisioning_task.rb

@@ -1,4 +1,5 @@
 require 'fhcap/tasks/cluster/cluster_task_base'
+require 'fhcap/tasks/chef/server/bootstrap'
 require 'fhcap/tasks/chef/provisioning/chef_provisioning_task'
 
 module Fhcap
@@ -7,6 +8,7 @@ module Fhcap
       class ChefProvisioningTask < ClusterTaskBase
 
         def run
+          Chef::Server::Bootstrap.new(@options.dup.merge({:environments => cluster_environments}.merge(chef_task_options))).run unless options[:'skip-server-bootstrap']
           Tasks::Chef::Provisioning::ChefProvisioningTask.new(@options.dup.merge(chef_task_options)).run
         end
 

data/lib/fhcap/tasks/cluster/create.rb

@@ -84,6 +84,7 @@ module Fhcap
                   options: {:limited_to => repo_names}
               },
               domain: {
+                msg: "Base domain/hostname (e.g. foo.skunkhenry.com) for this cluster?",
               },
               chef_server: {
                   options: {:limited_to => chef_server_names}
@@ -103,7 +104,9 @@ module Fhcap
               region: {
                   options: {:limited_to => aws_regions.keys.collect { |k| k.to_s }}
               },
-              cidr: {}
+              cidr: {
+                msg: "CIDR block for AWS VPC (e.g. 10.123.0.0/16)?",
+              }
           }
         end
 
@@ -241,4 +244,4 @@ module Fhcap
       end
     end
   end
-end
+end

data/lib/fhcap/tasks/cluster/create_environment.rb

@@ -160,6 +160,8 @@ module Fhcap
               'mbaas'
             when /single/
               'single'
+            when /farm/
+              'farm'
             else
               'empty'
           end

data/lib/fhcap/version.rb

@@ -1,3 +1,3 @@
 module Fhcap
-  VERSION = "0.4.3"
+  VERSION = "0.4.4"
 end

data/lib/fhcap.rb

@@ -3,7 +3,7 @@ require "pathname"
 
 module Fhcap
   GEM_DIR = File.expand_path '..', File.dirname(__FILE__)
-  TEMPLATE_NAMES = %w{single core-3node mbaas-3node core-mbaas-6node core-small-9node nginx-test single-blank}.sort
+  TEMPLATE_NAMES = %w{single core-3node mbaas-3node core-mbaas-6node core-small-9node nginx-test single-blank farm-3node farm-single}.sort
 
   class << self
     def source_root

data/spec/fhcap/tasks/chef/cookbook/update_metadata_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+require "fhcap/tasks/chef/cookbook/update_metadata"
+
+describe Fhcap::Tasks::Chef::Cookbook::UpdateMetadata do
+
+  subject {
+    Fhcap::Tasks::Chef::Cookbook::UpdateMetadata.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    {}
+  end
+
+  let(:options) do
+    {
+        :config => config,
+        :thor => thor,
+        :cookbook => 'testname'
+    }
+  end
+
+  describe "#initialize" do
+    specify { expect(subject.name).to eq('testname') }
+  end
+
+end

data/templates/chef/environment_core.json.erb

@@ -2,6 +2,7 @@
 require 'securerandom'
 
 @passwords = {}
+@keys = {}
 
 def random_password(name)
   unless @passwords[name]
@@ -9,6 +10,14 @@ def random_password(name)
   end
   @passwords[name]
 end
+
+def random_key(name)
+  unless @keys[name]
+    @keys[name] = SecureRandom.urlsafe_base64(16)
+  end
+  @keys[name]
+  
+end
 -%>
 {
   "name": "<%= config[:name] %>",
@@ -20,7 +29,15 @@ end
   "default_attributes": {
     "apache2": {
       "buildfarm": {
-        "url": "e103-stg-digman-01.feedhenry.net"
+        "android": {
+          "url": "qed1-builder.feedhenry.net"
+        },
+        "windowsphone": {
+          "url": "e103-stg-digman-01.feedhenry.net"
+        },
+        "ios": {
+          "url": "e103-stg-digman-01.feedhenry.net"
+        }
       }
     },
     "apache2_feedhenry": {
@@ -70,6 +87,7 @@ end
     },
     "fh-messaging": {
       "conf": {
+        "msgAPIKey":  "<%=random_key('fh-messaging')%>",
         "database": {
           "auth": {
             "user": "u-messaging",
@@ -85,6 +103,7 @@ end
     "fh-metrics": {
       "conf": {
         "metrics": {
+          "metricsAPIKey": "<%=random_key('fh-metrics')%>",  
           "database": {
             "auth": {
               "user": "u-messaging",
@@ -97,6 +116,12 @@ end
     "fh-supercore": {
       "conf": {
         "fhsupercore": {
+          "fhmetrics":{
+            "apikey":"<%=random_key('fh-metrics')%>"
+          },
+          "fhmessaging":{
+            "apikey":"<%=random_key('fh-messaging')%>"
+          },
           "database": {
             "auth": {
               "user": "u-supercore",
@@ -118,13 +143,9 @@ end
           }
         },
         "digger": {
-          "callbackhost": "127.0.0.1",
           "polling": {
             "iphone": true,
             "android": true
-          },
-          "download": {
-            "host": "e103-stg-digman-01.feedhenry.net"
           }
         },
         "ditch": {
@@ -186,6 +207,8 @@ end
         "studio-js-preview-default-device": "galaxy_s6",
         "studio-js-cloudresources-enabled": false,
         "studio-js-admin-environments-mbaases-ui": true,
+        "studio-protocol": "http",
+        "millicore.protocol": "http",
         "studio": {
           "version": {
             "option": {

data/templates/chef/environment_farm.json.erb

@@ -0,0 +1,60 @@
+<%-
+require 'securerandom'
+
+@passwords = {}
+
+def random_password(name)
+  unless @passwords[name]
+    @passwords[name]= SecureRandom.urlsafe_base64(6)
+  end
+  @passwords[name]
+end
+-%>
+{
+  "name": "<%= config[:name] %>",
+  "description": "<%= config[:name] %> Environment",
+  "cookbook_versions": {
+  },
+  "json_class": "Chef::Environment",
+  "chef_type": "environment",
+  "default_attributes": {
+    "apache2_feedhenry": {
+      "ops_proxy": {
+        "enabled": false
+      }
+    },
+    "feedhenry_common": {
+      "backups": {
+        "enabled": false
+      },
+      "domain": "<%= config[:domain] %>",
+      "baseurl": "<%= config[:name] %>.<%= config[:domain] %>",
+      "manage": {
+        "denyhosts": false
+      },
+      "notifications": {
+        "enabled": false,
+        "email": "root"
+      }
+    },
+    "nagios": {
+      "services_databag": "nagios_services,nagios_services_<%= config[:name].gsub('-', '_') %>",
+      "contacts_databag": "nagios_contacts",
+      "notifications_enabled": 0
+    }
+  },
+  "override_attributes": {
+    "authorization": {
+      "sudo": {
+        "passwordless": true
+      }
+    },
+    "denyhosts": {
+      "allowed_hosts": [],
+      "config": {
+        "admin_email": "",
+        "block_service": ""
+      }
+    }
+  }
+}

data/templates/chef/environment_mbaas.json.erb

@@ -2,13 +2,21 @@
 require 'securerandom'
 
 @passwords = {}
-
+@keys = {}
 def random_password(name)
   unless @passwords[name]
     @passwords[name]= SecureRandom.urlsafe_base64(6)
   end
   @passwords[name]
 end
+
+def random_key(name)
+  unless @keys[name]
+    @keys[name] = SecureRandom.urlsafe_base64(16)
+  end
+  @keys[name]
+  
+end
 -%>
 {
   "name": "<%= config[:name] %>",
@@ -62,9 +70,27 @@ end
           "dynofarm": "https://api.<%= config[:domain] %>",
           "username": "df-<%= config[:name] %>",
           "_password": "<%= random_password('fh-proxy') %>"
+        },
+        "fhmetrics":{
+          "apikey": "<%=random_key('fh-metrics')%>"
+        },
+        "fhmessaging":{
+          "apikey": "<%=random_key('fh-messaging')%>"
         }
       }
     },
+    "fh-metrics":{
+      "conf":{
+        "metrics": {
+          "metricsAPIKey": "<%=random_key('fh-metrics')%>"
+        }
+      }
+    },
+    "fh-messaging":{
+      "conf":{
+        "msgAPIKey":  "<%=random_key('fh-messaging')%>"
+      }
+    },
     "fh-proxy": {
       "conf": {
         "fhproxy": {

data/templates/chef/environment_single.json.erb

@@ -20,7 +20,15 @@ end
   "default_attributes": {
     "apache2": {
       "buildfarm": {
-        "url": "e103-stg-digman-01.feedhenry.net"
+        "android": {
+          "url": "qed1-builder.feedhenry.net"
+        },
+        "windowsphone": {
+          "url": "e103-stg-digman-01.feedhenry.net"
+        },
+        "ios": {
+          "url": "e103-stg-digman-01.feedhenry.net"
+        }
       }
     },
     "apache2_feedhenry": {
@@ -175,13 +183,9 @@ end
           }
         },
         "digger": {
-          "callbackhost": "127.0.0.1",
           "polling": {
             "iphone": true,
             "android": true
-          },
-          "download": {
-            "host": "e103-stg-digman-01.feedhenry.net"
           }
         },
         "ditch": {

data/templates/cluster/aws/farm-3node.json.erb

@@ -0,0 +1,104 @@
+{
+  "domain": "<%= config[:domain] %>",
+  "subnets": {
+    "1a": {
+      "cidr": "0/26",
+      "availability_zone": "1a"
+    },
+    "1b": {
+      "cidr": "64/26",
+      "availability_zone": "1b"
+    },
+    "1c": {
+      "cidr": "128/26",
+      "availability_zone": "1c"
+    }
+  },
+  "security_groups": {
+    "<%= config[:environment_name] %>-ssh": {
+      "authorize_ingress": [
+        {
+          "protocols": ["tcp"],
+          "start": 22,
+          "end": 22,
+          "sources": ["0.0.0.0/0"]
+        }
+      ]
+    },
+    "<%= config[:environment_name] %>-builder-lb": {
+      "authorize_ingress": [
+        {
+          "protocols": ["tcp"],
+          "start": 443,
+          "end": 443,
+          "sources": ["0.0.0.0/0"]
+        }
+      ]
+    },
+    "<%= config[:environment_name] %>": {
+      "authorize_ingress": [
+        {
+          "protocols": ["all"],
+          "start": 0,
+          "end": 65535,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["icmp"],
+          "start": -1,
+          "end": -1,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["tcp"],
+          "start": 80,
+          "end": 80,
+          "groups": ["<%= config[:environment_name] %>-builder-lb"]
+        }
+      ]
+    }
+  },
+  "load_balancers": {
+    "digman": {
+      "security_groups": ["<%= config[:environment_name] %>-builder-lb"],
+      "subnets": ["1a", "1b", "1c"],
+      "scheme": "internet-facing",
+      "listeners": [
+        {
+          "port": 80,
+          "protocol": "http",
+          "instance_port": 80,
+          "instance_protocol": "http"
+        }
+      ]
+    }
+  },
+  "instances": {
+    "linux1": {
+      "aws": {
+        "security_groups": ["ops-admin", "<%= config[:environment_name] %>", "<%= config[:environment_name] %>-ssh"],
+        "private_ip_address": "",
+        "subnet": "1a"
+      },
+      "load_balancers": [],
+      "run_list": ["role[mount_data_volumes]", "role[fh-digger]", "recipe[feedhenry_common::fhctl]"]
+    },
+    "digman1": {
+      "aws": {
+        "security_groups": ["ops-admin", "<%= config[:environment_name] %>", "<%= config[:environment_name] %>-ssh"],
+        "private_ip_address": "",
+        "subnet": "1b"
+      },
+      "load_balancers": ["digman"],
+      "run_list": ["role[fh-digman]"]
+    },
+    "mgt1": {
+      "aws": {
+        "security_groups": ["ops-admin", "<%= config[:environment_name] %>", "<%= config[:environment_name] %>-ssh"],
+        "private_ip_address": "",
+        "subnet": "1c"
+      },
+      "run_list": ["role[management_server]"]
+    }
+  }
+}

data/templates/cluster/aws/farm-single.json.erb

@@ -0,0 +1,87 @@
+{
+  "domain": "<%= config[:domain] %>",
+  "subnets": {
+    "1a": {
+      "cidr": "0/26",
+      "availability_zone": "1a"
+    },
+    "1b": {
+      "cidr": "64/26",
+      "availability_zone": "1b"
+    },
+    "1c": {
+      "cidr": "128/26",
+      "availability_zone": "1c"
+    }
+  },
+  "security_groups": {
+    "<%= config[:environment_name] %>-ssh": {
+      "authorize_ingress": [
+        {
+          "protocols": ["tcp"],
+          "start": 22,
+          "end": 22,
+          "sources": ["0.0.0.0/0"]
+        }
+      ]
+    },
+    "<%= config[:environment_name] %>-builder-lb": {
+      "authorize_ingress": [
+        {
+          "protocols": ["tcp"],
+          "start": 443,
+          "end": 443,
+          "sources": ["0.0.0.0/0"]
+        }
+      ]
+    },
+    "<%= config[:environment_name] %>": {
+      "authorize_ingress": [
+        {
+          "protocols": ["all"],
+          "start": 0,
+          "end": 65535,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["icmp"],
+          "start": -1,
+          "end": -1,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["tcp"],
+          "start": 80,
+          "end": 80,
+          "groups": ["<%= config[:environment_name] %>-builder-lb"]
+        }
+      ]
+    }
+  },
+  "load_balancers": {
+    "digman": {
+      "security_groups": ["<%= config[:environment_name] %>-builder-lb"],
+      "subnets": ["1a", "1b", "1c"],
+      "scheme": "internet-facing",
+      "listeners": [
+        {
+          "port": 80,
+          "protocol": "http",
+          "instance_port": 80,
+          "instance_protocol": "http"
+        }
+      ]
+    }
+  },
+  "instances": {
+    "linux1": {
+      "aws": {
+        "security_groups": ["ops-admin", "<%= config[:environment_name] %>", "<%= config[:environment_name] %>-ssh"],
+        "private_ip_address": "",
+        "subnet": "1a"
+      },
+      "load_balancers": ["digman"],
+      "run_list": ["role[mount_data_volumes]", "role[fh-digger]", "role[fh-digman]", "role[management_server]", "recipe[feedhenry_common::fhctl]"]
+    }
+  }
+}

data/templates/cluster/aws/mbaas-3node.json.erb

@@ -92,7 +92,7 @@
         "subnet": "1a"
       },
       "load_balancers": ["api"],
-      "run_list": ["role[mount_data_volumes]", "role[mongo_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[mount_data_volumes]", "role[mongo_server]", "role[rabbitmq_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
     },
     "ship2": {
       "aws": {
@@ -101,7 +101,7 @@
         "subnet": "1b"
       },
       "load_balancers": ["api"],
-      "run_list": ["role[mount_data_volumes]", "role[mongo_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[mount_data_volumes]", "role[mongo_server]", "role[rabbitmq_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
     },
     "mgt1": {
       "aws": {

data/templates/cluster/farm-3node.json.erb

@@ -0,0 +1,8 @@
+{
+  "environments": [
+    {
+      "name": "farm",
+      "template": "farm-3node"
+    }
+  ]
+}

data/templates/cluster/farm-single.json.erb

@@ -0,0 +1,8 @@
+{
+  "environments": [
+    {
+      "name": "farm",
+      "template": "farm-single"
+    }
+  ]
+}

data/templates/cluster/openstack/core-3node.json.erb

@@ -2,13 +2,13 @@
   "domain": "<%= config[:domain] %>",
   "instances": {
     "node1": {
-      "run_list": ["role[platform_mongo_server]", "role[rabbitmq_server]", "role[app_server]", "role[fh-scm]", "role[fh-messaging]", "role[fh-metrics]", "role[fh-stats]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[platform_mongo_server]", "role[rabbitmq_server]", "role[galera_server]", "role[app_server]", "role[fh-scm]", "role[fh-messaging]", "role[fh-metrics]", "role[fh-stats]", "recipe[feedhenry_common::fhctl]"]
     },
     "node2": {
-      "run_list": ["role[platform_mongo_server]", "role[rabbitmq_server]", "role[app_server]", "role[fh-scm]", "role[fh-messaging]", "role[fh-metrics]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[platform_mongo_server]", "role[rabbitmq_server]", "role[galera_server]", "role[app_server]", "role[fh-scm]", "role[fh-messaging]", "role[fh-metrics]", "recipe[feedhenry_common::fhctl]"]
     },
     "mgt1": {
-      "run_list": ["role[platform_mongo_server]", "role[mysql_server]", "role[gitlab-shell_server]", "role[management_server]", "recipe[feedhenry_common::fhctl]", "recipe[nginx_feedhenry::loadbalancer]"]
+      "run_list": ["role[platform_mongo_server]", "role[gitlab-shell_server]", "role[management_server]", "recipe[feedhenry_common::fhctl]", "recipe[nginx_feedhenry::loadbalancer]"]
     }
   }
 }

data/templates/cluster/openstack/farm-3node.json.erb

@@ -0,0 +1,14 @@
+{
+  "domain": "<%= config[:domain] %>",
+  "instances": {
+    "linux1": {
+      "run_list": ["role[fh-digger]", "recipe[feedhenry_common::fhctl]"]
+    },
+    "digman1": {
+      "run_list": ["role[fh-digman]"]
+    },
+    "mgt1": {
+      "run_list": ["role[management_server]"]
+    }
+  }
+}

data/templates/cluster/openstack/farm-single.json.erb

@@ -0,0 +1,8 @@
+{
+  "domain": "<%= config[:domain] %>",
+  "instances": {
+    "linux1": {
+      "run_list": ["role[fh-digger]", "role[fh-digman]", "role[management_server]", "recipe[feedhenry_common::fhctl]"]
+    }
+  }
+}

data/templates/cluster/openstack/mbaas-3node.json.erb

@@ -2,13 +2,13 @@
   "domain": "<%= config[:domain] %>",
   "instances": {
     "ship1": {
-      "run_list": ["role[mongo_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[mongo_server]", "role[rabbitmq_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
     },
     "ship2": {
-      "run_list": ["role[mongo_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[mongo_server]", "role[rabbitmq_server]", "role[dyno_server]", "role[fh-ditch]", "recipe[feedhenry_common::fhctl]"]
     },
     "mgt1": {
-      "run_list": ["role[platform_mongo_server]", "role[mongo_server]", "role[fh-stats]", "role[management_server]", "recipe[nginx_feedhenry::loadbalancer]"]
+      "run_list": ["role[mongo_server]", "role[fh-stats]", "role[management_server]", "recipe[nginx_feedhenry::loadbalancer]"]
     }
   }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fhcap-cli
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Michael Nairn
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-04 00:00:00.000000000 Z
+date: 2016-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -52,6 +52,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.5.2
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.7.0
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -365,10 +379,13 @@ files:
 - lib/cookbooks/provision/recipes/cluster_status.rb
 - lib/cookbooks/provision/recipes/common.rb
 - lib/cookbooks/provision/recipes/default.rb
+- lib/cookbooks/provision/recipes/galera_reset_cluster.rb
+- lib/cookbooks/provision/recipes/mongo_reset_cluster.rb
 - lib/cookbooks/provision/recipes/openstack.rb
 - lib/cookbooks/provision/recipes/openstack_cluster_create.rb
 - lib/cookbooks/provision/recipes/openstack_cluster_destroy.rb
-- lib/cookbooks/provision/recipes/reset_rabbitmq.rb
+- lib/cookbooks/provision/recipes/post_create_instances.rb
+- lib/cookbooks/provision/recipes/rabbitmq_reset_cluster.rb
 - lib/cookbooks/provision/recipes/restart_services.rb
 - lib/cookbooks/provision/recipes/seed_cookbooks.rb
 - lib/extensions/chef/provisioning.rb
@@ -455,6 +472,7 @@ files:
 - spec/fhcap/tasks/chef/cookbook/list_artifacts_spec.rb
 - spec/fhcap/tasks/chef/cookbook/update_artifact_spec.rb
 - spec/fhcap/tasks/chef/cookbook/update_config_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_metadata_spec.rb
 - spec/fhcap/tasks/cluster/create_spec.rb
 - spec/fhcap/tasks/dns/create_record_spec.rb
 - spec/fhcap/tasks/dns/delete_record_spec.rb
@@ -473,11 +491,14 @@ files:
 - templates/chef/cookbook/metadata.erb
 - templates/chef/environment_core.json.erb
 - templates/chef/environment_empty.json.erb
+- templates/chef/environment_farm.json.erb
 - templates/chef/environment_mbaas.json.erb
 - templates/chef/environment_single.json.erb
 - templates/cluster/aws/common.json.erb
 - templates/cluster/aws/core-3node.json.erb
 - templates/cluster/aws/core-small-9node.json.erb
+- templates/cluster/aws/farm-3node.json.erb
+- templates/cluster/aws/farm-single.json.erb
 - templates/cluster/aws/mbaas-3node.json.erb
 - templates/cluster/aws/nginx-test.json.erb
 - templates/cluster/aws/single-blank.json.erb
@@ -485,11 +506,15 @@ files:
 - templates/cluster/core-3node.json.erb
 - templates/cluster/core-mbaas-6node.json.erb
 - templates/cluster/core-small-9node.json.erb
+- templates/cluster/farm-3node.json.erb
+- templates/cluster/farm-single.json.erb
 - templates/cluster/mbaas-3node.json.erb
 - templates/cluster/nginx-test.json.erb
 - templates/cluster/openstack/common.json.erb
 - templates/cluster/openstack/core-3node.json.erb
 - templates/cluster/openstack/core-small-9node.json.erb
+- templates/cluster/openstack/farm-3node.json.erb
+- templates/cluster/openstack/farm-single.json.erb
 - templates/cluster/openstack/mbaas-3node.json.erb
 - templates/cluster/openstack/nginx-test.json.erb
 - templates/cluster/openstack/single-blank.json.erb
@@ -532,6 +557,7 @@ test_files:
 - spec/fhcap/tasks/chef/cookbook/list_artifacts_spec.rb
 - spec/fhcap/tasks/chef/cookbook/update_artifact_spec.rb
 - spec/fhcap/tasks/chef/cookbook/update_config_spec.rb
+- spec/fhcap/tasks/chef/cookbook/update_metadata_spec.rb
 - spec/fhcap/tasks/cluster/create_spec.rb
 - spec/fhcap/tasks/dns/create_record_spec.rb
 - spec/fhcap/tasks/dns/delete_record_spec.rb

data/lib/cookbooks/provision/recipes/reset_rabbitmq.rb

@@ -1,58 +0,0 @@
-include_recipe 'provision::common'
-include_recipe "provision::#{node['driver']}"
-
-cluster_config = cluster_config_for(node)
-org_name = cluster_config[:id]
-
-def rabbit_reset_required?(nodes)
-  # Need better logic to check whether or not a reset of rabbit is actually required
-  nodes.length > 1
-end
-
-default_instance_options = cluster_config[:default_instance_options]
-cluster_config[:environments].each do |env, env_config|
-  chef_environment = fh_name_for(org_name, env)
-
-  instances = {}
-  env_config[:instances].each do |instance_name, instance_config|
-    instances[instance_name_for(org_name, env, instance_name)] = {}
-    instances[instance_name_for(org_name, env, instance_name)][:instance_config] = instance_config
-    instances[instance_name_for(org_name, env, instance_name)][:machine_options] = machine_options_for(cluster_config[:driver], org_name, env, default_instance_options.merge(instance_config[cluster_config[:driver].to_sym] || {}))
-  end
-
-  rabbit_nodes = search(:node, "chef_environment:#{chef_environment} AND roles:rabbitmq_server").map { |n| n.name }
-
-  if rabbit_reset_required? rabbit_nodes
-    with_chef_environment chef_environment do
-
-      rabbit_nodes.each do |instance_name|
-        machine_execute 'sudo rabbitmqctl stop_app' do
-          machine instance_name
-        end
-
-        machine_execute 'sudo rabbitmqctl reset' do
-          machine instance_name
-        end
-
-        machine_execute 'sudo rabbitmqctl start_app' do
-          machine instance_name
-        end
-      end
-
-      machine_batch do
-        rabbit_nodes.each do |instance_name|
-          machine_options = instances[instance_name][:machine_options]
-          instance_config = instances[instance_name][:instance_config]
-          machine instance_name do
-            chef_environment chef_environment
-            run_list instance_config[:run_list]
-            machine_options(machine_options)
-          end
-        end
-      end
-
-    end
-  else
-    Chef::Log.info('Rabbitmq Reset not required!')
-  end
-end