fhcap-cli 0.4.5 → 0.4.6

data/lib/fhcap/tasks/cluster/create_dns_records.rb

@@ -0,0 +1,108 @@
+require 'fhcap/tasks/cluster/cluster_task_base'
+require "fhcap/tasks/dns/create_record"
+
+module Fhcap
+  module Tasks
+    module Cluster
+      class CreateDNSRecords < ClusterTaskBase
+
+        def initialize(options)
+          super(options)
+        end
+
+        def run
+          thor.say "Cluster::CreateDNSRecords", :yellow
+          create_dns_record
+        end
+
+        private
+
+        def create_dns_record
+          send(:"create_dns_record_#{cluster_config[:driver]}")
+        end
+
+        def create_dns_record_aws
+          cluster_config[:environments].each do |env_name, env_cfg|
+            env_cfg[:load_balancers].each do |lb_name, lb_cfg|
+              if lb_cfg[:scheme] == "internet-facing"
+                begin
+                  require 'aws-sdk'
+
+                  cfg = provider_config(cluster_config[:provider_id])
+                  creds = cfg[:credentials]
+
+                  elb = Aws::ElasticLoadBalancing::Client.new(
+                      region: cluster_config[:provider_config][:region],
+                      access_key_id: creds[:'aws-access-key'],
+                      secret_access_key: creds[:'aws-secret-key']
+                  )
+
+                  lb = load_balancer_name_for(name, env_name, lb_name)
+                  resp = elb.describe_load_balancers({
+                                                         load_balancer_names: [lb],
+                                                         page_size: 1,
+                                                     })
+
+                  elb = resp.load_balancer_descriptions.first
+                  thor.say "Found ELB name: #{elb.load_balancer_name}, dns_name: #{elb.dns_name}, zone_id: #{elb.canonical_hosted_zone_name_id}"
+
+                  dns_record_cfg = {
+                      domain: "*.#{env_cfg[:domain]}",
+                      :"alias-target" => {
+                          dns_name: elb.dns_name,
+                          hosted_zone_id: elb.canonical_hosted_zone_name_id
+                      }
+                  }
+                  Dns::CreateRecord.new(@options.dup.merge(dns_record_cfg)).run
+                rescue Aws::ElasticLoadBalancing::Errors::LoadBalancerNotFound => e
+                  thor.say_status 'error', "LoadBalancer #{lb} not found, unable to create DNS Entry", :red
+                rescue Aws::ElasticLoadBalancing::Errors::ServiceError => e
+                  thor.say_status 'error', e.message, :red
+                end
+              end
+            end if env_cfg[:load_balancers]
+          end
+        end
+
+        def create_dns_record_openstack
+          cluster_config[:environments].each do |env_name, env_cfg|
+            env_name = "#{name}-#{env_name}"
+            knife_config_file = knife_config_file_for(cluster_config[:chef_server])
+            #ToDo [RHMAP-2898] Use knife object
+            nodes = JSON.parse(`knife search "chef_environment:#{env_name} AND recipes:nginx_feedhenry\\:\\:loadbalancer" -c #{knife_config_file} -F json -a name -a cloud.public_ipv4`)
+
+            lb_node = nodes['rows'].collect do |row|
+              name, attrs = row.first
+              attrs
+            end.first
+
+            if lb_node
+              if lb_node['cloud.public_ipv4']
+                dns_record_cfg = {
+                    domain: "*.#{env_cfg[:domain]}",
+                    ipaddress: lb_node['cloud.public_ipv4']
+                }
+                Dns::CreateRecord.new(@options.dup.merge(dns_record_cfg)).run
+              else
+                thor.say "Found lb node '#{lb_node['name']}', but was unable to retrieve it's IP!!}"
+              end
+            else
+              thor.say "Unable to locate lb node in cluster!!"
+            end
+          end
+        end
+
+        def load_balancer_name_for(org_name, environment, lb_name)
+          fh_name_for('fh-lb', org_name, environment, lb_name)[0..31] #elb names have a max length of 32
+        end
+
+        def fh_name_for(*args)
+          args.collect do |str|
+            str.to_s.split('-')
+          end.flatten.uniq.join('-')
+        end
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/cluster/create_environment.rb

@@ -1,6 +1,7 @@
 require 'fhcap/tasks/cluster/cluster_task_base'
 require 'fhcap/tasks/chef/environments/create'
 require "fhcap/tasks/misc/create_ssl_cert"
+require 'erubis'
 
 module Fhcap
   module Tasks

data/lib/fhcap/tasks/cluster/generate.rb

@@ -0,0 +1,135 @@
+require 'fhcap/tasks/cluster/cluster_task_base'
+require 'fhcap/tasks/cluster/create_environment'
+require 'fhcap/tasks/chef/environments/create'
+
+module Fhcap
+  module Tasks
+    module Cluster
+      class Generate < ClusterTaskBase
+
+        attr_reader :name, :cluster_config
+
+        def initialize(options)
+          super(options, false)
+          @template = options[:template]
+          @repo = options[:'repo']
+          @provider_config = options[:'provider-config'] || {}
+          @domain = options[:domain]
+          @ssl_cert = options[:'ssl-cert']
+          @dns_provider_id = options[:'dns-provider-id']
+          @chef_server = options[:'chef-server']
+          @provider_id = options[:'provider-id']
+
+          @cluster_config = {
+              id: @name,
+              template: @template,
+              driver: "",
+              domain: @domain,
+              repo: @repo,
+              chef_server: @chef_server,
+              provider_id: @provider_id
+          }.merge(@cluster_config)
+
+          @cluster_config[:provider_config] = @cluster_config[:provider_config] || {}
+          @cluster_config[:provider_config].merge!(@provider_config)
+        end
+
+        def run
+          thor.say "Cluster::Generate: name = #{@name}", :yellow
+
+          #Generate cluster config
+          generate_cluster_config
+
+          #Create environment files
+          generate_cluster_environments
+
+          #Write config to disk
+          cluster_file = find_cluster(name) || File.join(repo_dir(cluster_config[:repo]), repo_clusters_dir(cluster_config[:repo]), "#{name}.json")
+          thor.create_file(cluster_file, JSON.pretty_generate(cluster_config))
+        end
+
+        private
+
+        def required_config
+          {
+              repo: {
+                  msg: "In which repo should this clusters config and environment files be stored?",
+                  options: {:limited_to => repo_names}
+              },
+              domain: {
+                msg: "Base domain/hostname (e.g. foo.skunkhenry.com) for this cluster?",
+              },
+              chef_server: {
+                  options: {:limited_to => chef_server_names}
+              },
+              provider_id: {
+                  options: {:limited_to => provider_names_for('compute')}
+              },
+              template: {
+                  options: {:limited_to => cluster_template_names}
+              }
+          }
+        end
+
+        def aws_required_config
+          aws_regions = provider_config(@cluster_config[:provider_id])[:regions]
+          {
+              region: {
+                  options: {:limited_to => aws_regions.keys.collect { |k| k.to_s }}
+              },
+              cidr: {
+                msg: "CIDR block for AWS VPC (e.g. 10.123.0.0/16)?",
+              }
+          }
+        end
+
+        def generate_cluster_config
+          ask_config(required_config, cluster_config)
+          cluster_config[:driver] = provider_type(cluster_config[:provider_id])
+          send(:"generate_cluster_config_#{cluster_config[:driver]}")
+          driver_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', cluster_config[:driver], "common.json.erb")
+          driver_template_config = template_as_object(driver_template_file, cluster_config)
+          cluster_config.merge!(driver_template_config)
+        end
+
+        def generate_cluster_config_aws
+          ask_config(aws_required_config, cluster_config[:provider_config])
+          aws_regions = provider_config(cluster_config[:provider_id])[:regions]
+          cluster_config[:default_instance_options] = cluster_config[:default_instance_options] || {}
+          cluster_config[:default_instance_options][:image_id] = aws_regions[cluster_config[:provider_config][:region].to_sym][:base_image]
+        end
+
+        def generate_cluster_config_openstack
+          cluster_config[:default_instance_options] = cluster_config[:default_instance_options] || {}
+          cluster_config[:default_instance_options][:image_ref] = provider_config(@cluster_config[:provider_id])[:image_ref]
+          cluster_config[:default_instance_options][:flavor_ref] = provider_config(@cluster_config[:provider_id])[:flavor_ref]
+          cluster_config[:default_instance_options][:floating_ip_pool] = provider_config(@cluster_config[:provider_id])[:floating_ip_pool]
+          cluster_config[:default_instance_options][:ssh_username] = provider_config(@cluster_config[:provider_id])[:ssh_username]
+        end
+
+        def generate_cluster_environments
+          env_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', "#{cluster_config[:template]}.json.erb")
+          template_config = template_as_object(env_template_file, cluster_config)
+
+          template_config[:environments].each do |env|
+            env_template_file = File.join(Fhcap.source_root, 'templates', 'cluster', cluster_config[:driver], "#{env[:template]}.json.erb")
+            template_config = template_as_object(env_template_file, cluster_config)
+
+            env_domain = [env[:'domain-prefix'], cluster_config[:domain]].compact.join('.')
+            env_cfg = {
+                :name => name,
+                :'environment-name' => env[:name],
+                :domain => env_domain,
+                :template => env[:template],
+                :'cluster-config' => cluster_config,
+                :'skip-create-cluster-file' => true
+            }
+
+            CreateEnvironment.new(@options.dup.merge(env_cfg)).run
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/fhcap/tasks/provider/add.rb

@@ -10,6 +10,7 @@ module Fhcap
 
         AWS_REQUIRED_CREDENTIALS = %w{aws-access-key aws-secret-key}
         OS_REQUIRED_CREDENTIALS = %w{os-auth-url os-tenant-id os-tenant-name os-username os-password}
+        DNSMADEEASY_REQUIRED_CREDENTIALS = %w{api-key secret-key}
 
         def initialize(options)
           super
@@ -20,6 +21,8 @@ module Fhcap
             @required_credentials = AWS_REQUIRED_CREDENTIALS
           elsif options[:type] == 'openstack'
             @required_credentials = OS_REQUIRED_CREDENTIALS
+          elsif options[:type] == 'dnsmadeeasy'
+            @required_credentials = DNSMADEEASY_REQUIRED_CREDENTIALS
           end
         end
 
@@ -129,6 +132,14 @@ module Fhcap
           cfg[:provides] << 'dns'
           cfg[:provides].uniq!
         end
+        
+        def dnsmadeeasy_config(cfg)
+          creds = cfg[:credentials]
+          domains = thor.ask("Domains: ").split(' ')
+          cfg[:domains] = domains
+          cfg[:provides] << 'dns'
+          cfg[:provides].uniq!
+        end
 
       end
     end

data/lib/fhcap/version.rb

@@ -1,3 +1,3 @@
 module Fhcap
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
 end

data/spec/fhcap/tasks/cluster/create_dns_records_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+require "fhcap/tasks/cluster/create_dns_records"
+
+describe Fhcap::Tasks::Cluster::CreateDNSRecords do
+
+  subject {
+    Fhcap::Tasks::Cluster::CreateDNSRecords.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    DummyConfig.new
+  end
+
+  let(:options) do
+    {
+        :config => config,
+        :thor => thor,
+        :name => 'testcluster',
+        :'cluster-config' => {
+            :id => 'testcluster',
+            :repo => 'testrepo',
+            :template => 'template',
+            :domain => 'testdomain',
+            :chef_server => 'test',
+            :provider_id => 'test'
+        }
+    }
+  end
+
+  describe "#initialize" do
+    specify { expect(subject.name).to eq(options[:name]) }
+  end
+
+  context "missing cluster config" do
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :name => 'testcluster'
+      }
+    end
+
+    describe "#initialize" do
+      it "should raise error and exit" do
+        expect { subject }.to raise_error(SystemExit)
+      end
+
+      it "should output correct error message" do
+        expect {
+          begin
+            subject
+          rescue SystemExit
+          end
+        }.to output(/Unknown cluster #{options[:name]}/).to_stdout
+      end
+
+    end
+  end
+
+end

data/spec/fhcap/tasks/cluster/create_spec.rb

@@ -20,27 +20,48 @@ describe Fhcap::Tasks::Cluster::Create do
         :config => config,
         :thor => thor,
         :name => 'testcluster',
-        :repo => 'testrepo',
-        :template => 'template',
         :'git-ref' => 'master',
-        :domain => 'testdomain',
-        :'ssl-cert' => 'testcert',
-        :'dns-provider-id' => 'test',
-        :'chef-server' => 'test',
-        :'provider-id' => 'test',
+        :'cluster-config' => {
+            :id => 'testcluster',
+            :repo => 'testrepo',
+            :template => 'template',
+            :domain => 'testdomain',
+            :chef_server => 'test',
+            :provider_id => 'test'
+        }
     }
   end
 
   describe "#initialize" do
     specify { expect(subject.name).to eq(options[:name]) }
     specify { expect(subject.instance_variable_get(:@gitref)).to eq(options[:'git-ref']) }
-    specify { expect(subject.instance_variable_get(:@ssl_cert)).to eq(options[:'ssl-cert']) }
-    specify { expect(subject.instance_variable_get(:@dns_provider_id)).to eq(options[:'dns-provider-id']) }
     specify { expect(subject.cluster_config[:id]).to eq(options[:name]) }
-    specify { expect(subject.cluster_config[:template]).to eq(options[:template]) }
-    specify { expect(subject.cluster_config[:domain]).to eq(options[:domain]) }
-    specify { expect(subject.cluster_config[:chef_server]).to eq(options[:'chef-server']) }
-    specify { expect(subject.cluster_config[:provider_id]).to eq(options[:'provider-id']) }
+  end
+
+  context "missing cluster config" do
+    let(:options) do
+      {
+          :config => config,
+          :thor => thor,
+          :name => 'testcluster',
+          :'git-ref' => 'master'
+      }
+    end
+
+    describe "#initialize" do
+      it "should raise error and exit" do
+        expect { subject }.to raise_error(SystemExit)
+      end
+
+      it "should output correct error message" do
+        expect {
+          begin subject
+          rescue SystemExit
+          end
+        }.to output(/Unknown cluster #{options[:name]}/).to_stdout
+      end
+
+    end
   end
 
 end

data/spec/fhcap/tasks/cluster/generate_spec.rb

@@ -0,0 +1,44 @@
+require 'spec_helper'
+require "fhcap/tasks/cluster/generate"
+
+describe Fhcap::Tasks::Cluster::Generate do
+
+  subject {
+    Fhcap::Tasks::Cluster::Generate.new(options)
+  }
+
+  let(:thor) do
+    DummyThor.new
+  end
+
+  let(:config) do
+    DummyConfig.new
+  end
+
+  let(:options) do
+    {
+        :config => config,
+        :thor => thor,
+        :name => 'testcluster',
+        :repo => 'testrepo',
+        :template => 'template',
+        :domain => 'testdomain',
+        :'ssl-cert' => 'testcert',
+        :'dns-provider-id' => 'test',
+        :'chef-server' => 'test',
+        :'provider-id' => 'test',
+    }
+  end
+
+  describe "#initialize" do
+    specify { expect(subject.name).to eq(options[:name]) }
+    specify { expect(subject.instance_variable_get(:@ssl_cert)).to eq(options[:'ssl-cert']) }
+    specify { expect(subject.instance_variable_get(:@dns_provider_id)).to eq(options[:'dns-provider-id']) }
+    specify { expect(subject.cluster_config[:id]).to eq(options[:name]) }
+    specify { expect(subject.cluster_config[:template]).to eq(options[:template]) }
+    specify { expect(subject.cluster_config[:domain]).to eq(options[:domain]) }
+    specify { expect(subject.cluster_config[:chef_server]).to eq(options[:'chef-server']) }
+    specify { expect(subject.cluster_config[:provider_id]).to eq(options[:'provider-id']) }
+  end
+
+end

data/templates/chef/environment_core.json.erb

@@ -36,7 +36,7 @@ end
           "url": "e103-stg-digman-01.feedhenry.net"
         },
         "ios": {
-          "url": "e103-stg-digman-01.feedhenry.net"
+          "url": "meh1-builder.feedhenry.net"
         }
       }
     },
@@ -60,6 +60,7 @@ end
       }
     },
     "feedhenry_galera": {
+      "cluster_enabled": true,
       "safe-updates": false,
       "wsrep": {
         "password": "<%= random_password('galerawsrep') %>"
@@ -222,8 +223,7 @@ end
       "auto_configure": {
         "replicaset": true
       },
-      "configfile": null,
-      "nojournal": true
+      "configfile": null
     },
     "mongodb_feedhenry": {
       "replicaset_enabled": true,
@@ -261,6 +261,11 @@ end
       "contacts_databag": "nagios_contacts",
       "notifications_enabled": 0
     },
+    "nagios_feedhenry": {
+      "api": {
+        "enabled": false
+      }
+    },
     "nginx_feedhenry": {
       "ssl_cert_item": "wildcard.<%= config[:domain] %>",
       "loadbalancer": "<%= config[:domain].gsub('.', '\\.') %>"

data/templates/chef/environment_farm.json.erb

@@ -41,6 +41,11 @@ end
       "services_databag": "nagios_services,nagios_services_<%= config[:name].gsub('-', '_') %>",
       "contacts_databag": "nagios_contacts",
       "notifications_enabled": 0
+    },
+    "nagios_feedhenry": {
+      "api": {
+        "enabled": false
+      }
     }
   },
   "override_attributes": {

data/templates/chef/environment_mbaas.json.erb

@@ -26,6 +26,11 @@ end
   "json_class": "Chef::Environment",
   "chef_type": "environment",
   "default_attributes": {
+    "apache2_feedhenry": {
+      "ops_proxy": {
+        "enabled": false
+      }
+    },
     "feedhenry_common": {
       "backups": {
         "enabled": false
@@ -106,8 +111,7 @@ end
       "auto_configure": {
         "replicaset": true
       },
-      "configfile": null,
-      "nojournal": true
+      "configfile": null
     },
     "mongodb_feedhenry": {
       "replicaset_enabled": true,
@@ -125,6 +129,11 @@ end
       "contacts_databag": "nagios_contacts",
       "notifications_enabled": 0
     },
+    "nagios_feedhenry": {
+      "api": {
+        "enabled": false
+      }
+    },
     "nginx_feedhenry": {
       "ssl_cert_item": "wildcard.<%= config[:domain] %>",
       "loadbalancer": "<%= config[:domain].gsub('.', '\\.') %>"

data/templates/chef/environment_single.json.erb

@@ -27,7 +27,7 @@ end
           "url": "e103-stg-digman-01.feedhenry.net"
         },
         "ios": {
-          "url": "e103-stg-digman-01.feedhenry.net"
+          "url": "meh1-builder.feedhenry.net"
         }
       }
     },
@@ -142,27 +142,31 @@ end
                 "url": "http://api.<%= config[:domain] %>",
                 "servicekey": "mBqxpulwlj3Rw7EsHZq0YEQyrArq2tlIRCQ0pCmr5DRPPDZMAP4ZNb7U0E6bPSb3",
                 "username": "df-<%= config[:name] %>",
-                "password": "feedhenry101"
+                "password": "feedhenry101",
+                "domain":"testing"
               }
             },
             "environments": {
               "dev": {
                 "label": "Development",
-                "targets": ["mbaas1"],
+                "target": "mbaas1",
                 "autoDeployOnCreate": true,
-                "autoDeployOnUpdate": false
+                "autoDeployOnUpdate": false,
+                "domain":"testing"
               },
               "test": {
                 "label": "Test",
-                "targets": ["mbaas1"],
+                "target": "mbaas1",
                 "autoDeployOnCreate": false,
-                "autoDeployOnUpdate": false
+                "autoDeployOnUpdate": false,
+                "domain":"testing"
               },
               "live": {
                 "label": "Production",
-                "targets": ["mbaas1"],
+                "target": "mbaas1",
                 "autoDeployOnCreate": false,
-                "autoDeployOnUpdate": false
+                "autoDeployOnUpdate": false,
+                "domain":"testing"
               }
             }
           }
@@ -282,6 +286,11 @@ end
       "contacts_databag": "nagios_contacts",
       "notifications_enabled": 0
     },
+    "nagios_feedhenry": {
+      "api": {
+        "enabled": false
+      }
+    },
     "nginx_feedhenry": {
       "loadbalancer": "<%= config[:domain].gsub('.', '\\.') %>"
     }

data/templates/cluster/aws/common.json.erb

@@ -35,7 +35,7 @@
           "protocols": ["all"],
           "start": 0,
           "end": 65535,
-          "sources": ["83.147.149.210/32", "46.38.161.225/32", "54.229.76.48/32", "79.125.117.182/32", "78.137.150.209/32"]
+          "sources": ["83.147.149.210/32", "46.38.161.225/32", "54.229.76.48/32", "79.125.117.182/32", "78.137.150.209/32", "52.37.106.23", "52.50.12.70", "52.62.158.176", "52.70.198.93", "52.193.17.19", "52.86.106.110", "52.48.49.57"]
         }
       ]
     },

data/templates/cluster/aws/core-3node.json.erb

@@ -37,7 +37,19 @@
           "protocols": ["all"],
           "start": 0,
           "end": 65535,
-          "groups": ["<%= config[:environment_name] %>", "<%= config[:environment_name] %>-studio-lb"]
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["icmp"],
+          "start": -1,
+          "end": -1,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["tcp"],
+          "start": 80,
+          "end": 80,
+          "groups": ["<%= config[:environment_name] %>-studio-lb"]
         }
       ]
     },
@@ -100,7 +112,7 @@
         "private_ip_address": "",
         "subnet": "1c"
       },
-      "run_list": ["role[platform_mongo_server]", "role[mysql_server]", "role[gitlab-shell_server]", "role[management_server]", "recipe[feedhenry_common::fhctl]"]
+      "run_list": ["role[mount_data_volumes]", "role[platform_mongo_server]", "role[galera_arbiter_server]", "role[gitlab-shell_server]", "role[management_server]", "recipe[feedhenry_common::fhctl]"]
     }
   }
 }

data/templates/cluster/aws/core-small-9node.json.erb

@@ -37,7 +37,19 @@
           "protocols": ["all"],
           "start": 0,
           "end": 65535,
-          "groups": ["core", "core-studio-lb"]
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["icmp"],
+          "start": -1,
+          "end": -1,
+          "groups": ["<%= config[:environment_name] %>"]
+        },
+        {
+          "protocols": ["tcp"],
+          "start": 80,
+          "end": 80,
+          "groups": ["<%= config[:environment_name] %>-studio-lb"]
         }
       ]
     },