avst-cloud 0.1.7

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+
+task :default => [:build]
+

data/avst-cloud.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+    spec.name          = "avst-cloud"
+    spec.version       = '0.1.7'
+    spec.authors       = ["Martin Brehovsky", "Jon Bevan", "Matthew Hope"]
+    spec.email         = ["mbrehovsky@adaptavist.com", "jbevan@adaptavist.com", "mhope@adaptavist.com"]
+    spec.summary       = %q{Automated creation, bootstrapping and provisioning of servers }
+    spec.description   = %q{Currently supports AWS}
+    spec.homepage      = "http://www.adaptavist.com"
+
+    spec.files         = `git ls-files -z`.split("\x0")
+    spec.executables   = ["avst-cloud", "avst-cloud-puppet", "avst-cloud-rackspace"]
+    spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+    spec.require_paths = ["lib"]
+
+    spec.add_development_dependency "bundler", "~> 1.6"
+    spec.add_development_dependency "rake"
+    spec.add_dependency "fog"
+    spec.add_dependency "capistrano", "3.2.1"
+    spec.add_dependency "capistrano-rvm"
+    spec.add_dependency "derelict"
+    spec.add_dependency "docopt", ">= 0.5.0"
+    spec.add_dependency "colorize", ">= 0.7.3"
+end
+

data/bin/avst-cloud

@@ -0,0 +1,99 @@
+#!/usr/bin/env ruby 
+
+# Copyright 2015 Adaptavist.com Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'avst-cloud'
+
+avst_cloud_base="#{File.expand_path("../../", __FILE__)}"
+
+# FILL IN YOUR AWS ACCOUNT DETAILS (Access Key ID and Secret Access Key), GIT REPO AND MAKE SURE 
+# FILES IN files FOLDER HAS CORRECT CONTENT
+provider_user=''
+provider_pass=''
+region='eu-west-1'
+# Initiate connector object
+conn = AvstCloud::AwsConnection.new(provider_user, provider_pass, region)
+
+server_name        = 'hostname1'
+
+flavour            = nil # defaults to t2.micro
+hdd_device_path    = nil # defaults to '/dev/sda1'
+ami_image_id       = "ami-f0b11187"
+os                 = "ubuntu-14"
+
+key_name           = 'admin' # key name, e.g. admin
+ssh_key            = "path/to/your/admin.pem" # Full path to ssh key, e.g. /tmp/admin.pem
+subnet_id          = '' # Your subnet
+security_group_ids = [''] # List of security groups
+ebs_size           = nil # In GB
+availability_zone  = 'eu-west-1c'
+
+server = conn.create_server(server_name, flavour, os, key_name, ssh_key, subnet_id, security_group_ids, ebs_size, hdd_device_path, ami_image_id, availability_zone)
+
+# server = conn.server(server_name, nil, ssh_key, os)
+# server.destroy
+# puts conn.server_status(server_name)
+# exit
+
+pre_upload_commands = [
+    "echo 'pre_upload_command was here' >> /tmp/pre_upload",
+]
+
+custom_file_uploads = {
+    "#{avst_cloud_base}/files/id_rsa" => "/tmp/.",
+    "#{avst_cloud_base}/files/known_hosts" => "/tmp/."
+}
+
+# Uploading ssh keys to access git repo in provisioning stage, make sure you provide correct keys
+post_upload_commands = [
+    "mkdir /home/ubuntu/.ssh",
+    "mv /tmp/id_rsa /home/ubuntu/.ssh/.",
+    "mv /tmp/known_hosts /home/ubuntu/.ssh/.",
+    "chmod 0600 /home/ubuntu/.ssh/known_hosts",
+    "chmod 0600 /home/ubuntu/.ssh/id_rsa",
+    "mkdir /var/opt/puppet",
+    "chown ubuntu /var/opt/puppet",
+    "apt-get update && apt-get install -o Dpkg::Options::='--force-confold' -f -y git puppet-common puppet"
+]
+
+remote_server_debug = true
+debug_structured_log = false
+
+server.bootstrap(pre_upload_commands, custom_file_uploads, post_upload_commands, remote_server_debug, debug_structured_log)
+
+git = "ssh://git@you_repo.git"
+branch = "master" 
+reference = nil # Tag
+# In this example we do not use puppet-runner, check doco
+puppet_runner = nil
+puppet_runner_prepare = nil
+custom_provisioning_commands = ["echo 'done' >> /tmp/done", "echo 'done' >> /tmp/done1"]
+server_tmp_folder="/tmp/avst_cloud_tmp_#{Time.now.to_i}"
+
+destination_folder = nil # defaults to /var/opt/puppet
+server.provision(git, branch, server_tmp_folder, reference, custom_provisioning_commands, puppet_runner, puppet_runner_prepare, destination_folder)
+
+# puts conn.server_status(server_name)
+# server.stop
+
+# puts conn.server_status(server_name)
+# server.start
+
+# custom_commands=nil
+# server_tmp_folder="/tmp/done"
+# server.post_provisioning_cleanup(custom_commands, os, remote_server_debug, server_tmp_folder)
+
+# server.destroy
+# puts conn.server_status(server_name)

data/bin/avst-cloud-puppet

@@ -0,0 +1,173 @@
+#!/usr/bin/env ruby 
+# Copyright 2015 Adaptavist.com Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'avst-cloud'
+
+avst_cloud_base ="#{File.expand_path("../../", __FILE__)}"
+
+start_time = Time.now
+
+server_name = "hostname1"
+
+# FILL IN YOUR AWS ACCOUNT DETAILS (Access Key ID and Secret Access Key), GIT REPO AND MAKE SURE 
+# FILES IN files FOLDER HAS CORRECT CONTENT
+
+# Provide account specific values here
+provider_user=''
+provider_pass=''
+
+key_name           = 'admin' # key name, e.g. admin
+ssh_key            = "path/to/your/admin.pem" # Full path to ssh key, e.g. /tmp/admin.pem
+subnet_id          = '' # Your subnet
+security_group_ids = [''] # List of security groups
+
+region             = "eu-west-1"
+
+conn = AvstCloud::AwsConnection.new(provider_user, provider_pass, region)
+
+ebs_size           = 12
+availability_zone  = "eu-west-1c"
+flavour            = "t2.small"
+
+# os specific, on aws ubuntu - '/dev/sda1', centos - '/dev/xvda'
+# hdd_device_path    = nil # defaults to '/dev/sda1'
+# ami_image_id       = "ami-f0b11187"
+# os                 = "ubuntu-14"
+# access_user        = 'ubuntu'
+
+hdd_device_path    = '/dev/xvda' # defaults to '/dev/sda1'
+ami_image_id       = "ami-7a40920d"
+os                 = "centos-7"
+access_user        = 'ec2-user'
+
+puts "Starting server".green
+
+server = conn.create_server(server_name, flavour, os, key_name, ssh_key, subnet_id, security_group_ids, ebs_size, hdd_device_path, ami_image_id, availability_zone)
+# server = conn.server(server_name, nil, ssh_key, os)
+# server.destroy
+# exit
+# # Ubuntu 14 pre upload commands
+# pre_upload_commands = [
+#    "echo 'pre_upload_command was here' >> /tmp/pre_upload",
+#    "apt-get clean",
+#    "apt-get update",
+#    "apt-get upgrade -o Dpkg::Options::='--force-confold' --force-overwrite -f -y",
+#    "apt-get install -o Dpkg::Options::='--force-confold' --force-overwrite -f -y curl",
+#    "curl -sSL https://raw.github.com/adaptavist/rvm/adaptavist_rvm/binscripts/rvm-installer | bash -s -- --version 1.25.33",
+#    "source /usr/local/rvm/scripts/rvm; rvm install 2.0",
+#    "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc",
+#    "apt-get install -o Dpkg::Options::='--force-confold' -f -y git puppet-common puppet augeas-tools libaugeas-dev libaugeas-ruby unzip",
+#    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc puppet -v 3.7.5",
+#    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc r10k ruby-augeas hiera-eyaml hiera-fragment ruby-shadow",
+#    "echo #{server_name} > /etc/hostname; sed -i 's/ubuntu/#{server_name}/g' /etc/hosts; sed -i '/127.0.1.1/d' /etc/hosts; echo '127.0.1.1 #{server_name}' >> /etc/hosts; service hostname restart",
+#    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc puppet-runner",
+#    "mkdir /home/#{access_user}/.ssh",
+#    "mkdir /var/opt/puppet/secrets/keys"
+#]
+
+# CentOS 7 pre upload commands
+pre_upload_commands = [
+    # clean all the YUM metadata
+    "yum clean all",
+    # update all packages already installed
+    "yum -y update",
+    # install rvm
+    "curl -sSL https://raw.github.com/adaptavist/rvm/adaptavist_rvm/binscripts/rvm-installer | bash -s -- --version 1.25.33",
+    # install ruby 2.0 via rvm
+    "source /usr/local/rvm/scripts/rvm; rvm install 2.0",
+    # add defaults to ~/.gemrc
+    "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc",
+    # install curl, git, puppet-common, puppet, augeas-devel, augeas and unzip
+    "yum install -y curl git puppet-common puppet augeas-devel augeas unzip",
+    # install puppet 3.7.5 gem
+    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc puppet -v 3.7.5",
+    # install r10k, ruby-augeas, hiera-eyaml, hiera-fragment and ruby-shadow gem
+    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc r10k ruby-augeas hiera-eyaml hiera-fragment ruby-shadow",
+    # set the servers hostname
+    "echo 'HOSTNAME=#{server_name}' >> /etc/sysconfig/network; sed -i '/127.0.1.1/d' /etc/hosts; echo '127.0.1.1 #{server_name}' >> /etc/hosts; hostname #{server_name};",
+    "if [ ! -f /usr/bin/hostnamectl ]; then  /etc/init.d/network restart; else /usr/bin/hostnamectl set-hostname #{server_name}; fi",
+    # workaround for issues resizing the root flesystem if its xfs
+    "mount -t xfs | egrep ' / '.*.inode64 > /dev/null 2>&1 && (mount -o remount,inode32 /;mount -o remount,inode64 /)",
+    # download and install oracle java 7 and 8, alternatives will be configured by the oracle_java puppet module
+    "curl -L -b 'oraclelicense=a' http://download.oracle.com/otn-pub/java/jdk/8u65-b17/jdk-8u65-linux-x64.rpm -o /tmp/jdk-8u65-linux-x64.rpm",
+    "curl -L -b 'oraclelicense=a' http://download.oracle.com/otn-pub/java/jdk/7u79-b15/jdk-7u79-linux-x64.rpm -o /tmp/jdk-7u79-linux-x64.rpm",
+    "rpm -i /tmp/jdk-8u65-linux-x64.rpm",
+    "rpm -i /tmp/jdk-7u79-linux-x64.rpm",
+    # install puppet-runer gem
+    "source /usr/local/rvm/scripts/rvm; gem install --no-ri --no-rdoc puppet-runner",
+    # create the access users .ssh folder to allow the uploading of id_rsa and known_hosts
+    "mkdir /home/#{access_user}/.ssh",
+    # create the puppet secrets directory to allow the uploading of eyaml public and private keys
+    "mkdir -p /var/opt/puppet/secrets/keys",
+    # ensure the acces user owns the puppet directories
+    "chown -R #{access_user} /var/opt/puppet",
+]
+
+custom_file_uploads = {
+    "#{avst_cloud_base}/files/id_rsa" => "/home/#{access_user}/.ssh/.",
+    "#{avst_cloud_base}/files/known_hosts" => "/home/#{access_user}/.ssh/.",
+    "#{avst_cloud_base}/files/private_key.pkcs7.pem" => "/tmp/.",
+    "#{avst_cloud_base}/files/public_key.pkcs7.pem" => "/tmp/."
+}
+
+# Uploading ssh keys to access git repo in provisioning stage, make sure you provide correct keys
+# # Ubuntu 14 post upload commands
+# post_upload_commands = [
+#     "chmod 0600 /home/#{access_user}/.ssh/known_hosts",
+#     "chmod 0600 /home/#{access_user}/.ssh/id_rsa",
+#     "mkdir /var/opt/puppet",
+#     "chown ubuntu /var/opt/puppet",
+#     "apt-get update && apt-get install -o Dpkg::Options::='--force-confold' -f -y git puppet-common puppet"
+# ]
+
+# Centos 7 post upload commands
+post_upload_commands = [
+    "mv /tmp/private_key.pkcs7.pem /var/opt/puppet/secrets/keys/.",
+    "mv /tmp/public_key.pkcs7.pem /var/opt/puppet/secrets/keys/.",
+    # download the mysql jdbc driver
+    "curl -L http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.22/mysql-connector-java-5.1.22.jar -o /var/opt/puppet/mysql-connector-java-5.1.22.jar",
+    "chmod 0600 /home/#{access_user}.ssh/known_hosts",
+    "chmod 0600 /home/#{access_user}/.ssh/id_rsa",
+    # install the puppet and puppet-common packages via yum
+    "yum install -y git puppet-common puppet"
+]
+
+
+remote_server_debug = true
+debug_structured_log = false
+
+server.bootstrap(pre_upload_commands, custom_file_uploads, post_upload_commands, remote_server_debug, debug_structured_log)
+
+server_tmp_folder = "/tmp/avst_cloud_tmp_#{Time.now.to_i}"
+
+# url to git repo you want to upload to the server
+git = "https://github.com/Adaptavist/base_puppet_templates.git" # url to git repo
+branch = "master" # branch
+reference = nil # tag
+# In this example we are using puppet-runner to apply our puppet configs, check doco
+puppet_runner = "puppet-runner start"
+puppet_runner_prepare = "puppet-runner prepare -c ./hiera-configs -d ./hiera -f ./environments/production/modules/hosts/facts.d -t ./hiera-configs -r ./hiera-configs/puppetfile_dictionary.yaml -o ./Puppetfile -e /var/opt/puppet/secure/keys"
+
+custom_provisioning_commands = ["echo 'done' >> /tmp/done"]
+# defaults to /var/opt/puppet
+destination_folder = nil
+
+server.provision(git, branch, server_tmp_folder, reference, custom_provisioning_commands, puppet_runner, puppet_runner_prepare, destination_folder)
+
+# Destroy server 
+# server.destroy
+
+time_taken = ((Time.now - start_time) / 60).round(2)
+puts "Finished in #{time_taken} minutes".green

data/bin/avst-cloud-rackspace

@@ -0,0 +1,87 @@
+#!/usr/bin/env ruby 
+
+# Copyright 2015 Adaptavist.com Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'avst-cloud'
+
+avst_cloud_base="#{File.expand_path("../../", __FILE__)}"
+
+# FILL IN YOUR Rackspace ACCOUNT DETAILS (Username and Secret Access Key), GIT REPO, image_id AND MAKE SURE 
+# FILES IN files FOLDER HAS CORRECT CONTENT
+provider_user=''
+provider_pass=''
+region=:lon
+# Initiate connector object
+conn = AvstCloud::RackspaceConnection.new(provider_user, provider_pass, region)
+
+server_name        = 'hostname1'
+
+flavour            = '4'
+image_id           = '' # Please provide
+server = conn.create_server(server_name, image_id)
+
+puts conn.server_status(server_name)
+# server = conn.server(server_name, 'root', pass)
+
+pre_upload_commands = [
+    "echo 'pre_upload_command was here' >> /tmp/pre_upload",
+]
+
+custom_file_uploads = {
+    "#{avst_cloud_base}/files/id_rsa" => "/tmp/.",
+    "#{avst_cloud_base}/files/known_hosts" => "/tmp/."
+}
+
+# Uploading ssh keys to access git repo in provisioning stage, make sure you provide correct keys
+post_upload_commands = [
+    "mkdir /root/.ssh",
+    "mv /tmp/id_rsa /root/.ssh/.",
+    "mv /tmp/known_hosts /root/.ssh/.",
+    "chmod 0600 /root/.ssh/known_hosts",
+    "chmod 0600 /root/.ssh/id_rsa",
+    "mkdir /var/opt/puppet",
+    "chown root /var/opt/puppet",
+    "apt-get update && apt-get install -o Dpkg::Options::='--force-confold' -f -y git puppet-common puppet"
+]
+
+remote_server_debug = true
+debug_structured_log = false
+
+server.bootstrap(pre_upload_commands, custom_file_uploads, post_upload_commands, remote_server_debug, debug_structured_log)
+
+git = "ssh://git@you_repo.git"
+branch = "master" 
+reference = nil # Tag
+# In this example we do not use puppet-runner, check doco
+puppet_runner = nil
+puppet_runner_prepare = nil
+custom_provisioning_commands = ["echo 'done' >> /tmp/done", "echo 'done' >> /tmp/done1"]
+server_tmp_folder="/tmp/avst_cloud_tmp_#{Time.now.to_i}"
+
+destination_folder = nil # defaults to /var/opt/puppet
+server.provision(git, branch, server_tmp_folder, reference, custom_provisioning_commands, puppet_runner, puppet_runner_prepare, destination_folder)
+
+# puts conn.server_status(server_name)
+# server.stop
+
+# puts conn.server_status(server_name)
+# server.start
+
+# custom_commands=nil
+# server_tmp_folder="/tmp/done"
+# server.post_provisioning_cleanup(custom_commands, os, remote_server_debug, server_tmp_folder)
+
+# server.destroy
+# puts conn.server_status(server_name)

data/config/custom_system_config/hiera-configs/defaults/example.yaml

@@ -0,0 +1 @@
+---

data/config/custom_system_config/hiera-configs/hostname1.yaml

@@ -0,0 +1,12 @@
+---
+functionalities:
+  0_base_setup:
+    - example
+    - oracle_java
+  1_app:
+    - jira: "atlassian_"
+  2_database:
+    - mysql: "atlassian_"
+  3_webserver:
+    - apache_http_atlassian_redhat_7 
+

data/config/custom_system_config/hiera-configs/hostname1_facts.yaml

@@ -0,0 +1,53 @@
+---
+
+# the jira version to install
+atlassian_version: '6.4.11'
+# the name of the jira instance, this will be the serice name and the installation will be in /opt/<INSTANCE_NAME>
+atlassian_instance_name: 'example_jira'
+# the jira administrator username 
+atlassian_admin_user: 'admin'
+# the jira administrator password
+atlassian_admin_pass: 'changeme'
+# the name of the mysql database to use
+atlassian_database_name: 'example_jira'
+# the mysql user for jira to use
+atlassian_database_user: 'example_jira'
+# the mysql password for jira to use
+atlassian_database_pass: 'changeme'
+# the mysql root users password
+atlassian_root_password: 'changeme'
+# the service user for jira
+atlassian_hosting_group: 'atlassian'
+# the service group for jiral
+atlassian_hosting_user: 'atlassian'
+# the jira license, the one below is a 3 hour timebomb license, for details see
+# https://developer.atlassian.com/market/add-on-licensing-for-developers/timebomb-licenses-for-testing
+atlassian_license: |
+    AAABiQ0ODAoPeNp1kk9TwjAQxe/9FJnxXKYpeoCZHqCtgsqfgaIO4yWELURD0tm0KN/eWOjYdvD68
+    vbtb3dzM9GKTBgS2iOU9n3a7/pkHiXE96jvbNhho3XnWXBQBuKtyIVWQTxN4sV8MV7GTirMHk5QO
+    ZJTBsG91eITvPdJBEeQOgN0uNRHwIYtLKWGa1ocNoCzdGUATUA9h2uVdhjPxRGCHAtw5gXyPTMQs
+    RwCn1Lf9XzXv3NqwVN2gGCZDBYWstLj70zgqSyad0fVWPXgJaClGUfB8KGXuG+rl1v3ab0euUOPv
+    jofAlmD/XG8GJBY5YAZCtMa9Ze5MagVZAGKX/FVE4eyMDZtqrdgAq+19zJlWEr/Na0TXjkTx4KLj
+    WzeKbyIjaAJE7aDYpa2tTSO+mvbCrBKo/ryate4Up9KfylnhjumhGEl0SCXzBjB1B9Q/QYhQulrH
+    /fcue6svl1di8BwFFnZKAGTE3mGIalGksliJxTZVqTmvLF6fXxksjhzpkwaqP5s3fMDBMYhRDAtA
+    hUAhcR3uL05YCxbclq7h1dNa+Nc+j4CFBrdN005oVlMN9yBlWeM4TlnrOhqX02j3
+# apache timeout for the proxy connection to jira, this is a large timeout as the setup wizard requires it
+vhost_http_atlassian_proxy_connection_timeout: 150
+vhost_http_atlassian_proxy_timeout: 300
+# hostname used to pass the setup wizard, will also be set as the jira base url
+atlassian_host_name: 'http://localhost'
+#JVM minimum memory setup for the service
+atlassian_jvm_minimum_memory: 256m
+#JVM maximum memory setup for the service
+atlassian_jvm_maximum_memory: 512m
+#JVM max perm size setup for the service
+atlassian_jvm_max_perm_size: 256m
+#the location of the mysql datbaase driver, it is downloaded as part of the example bootstrap
+atlassian_driver_location_path: '/var/opt/puppet/mysql-connector-java-5.1.22.jar'
+# the apache servername to set in the vhost
+vhost_http_atlassian_servername: 'hostname1'
+# the apache serveralias to set in the vhost
+vhost_http_atlassian_serveraliases: 'localhost'
+# the tomcat port that jira is running on
+connector_tomcat_port: '8081'
+