avst-cloud 0.1.7

data/config/custom_system_config/hiera-configs/templates/example.yaml

@@ -0,0 +1,137 @@
+---
+
+prefixes: 
+    - "example_"
+
+classes:
+  - baseusers
+  - packages_repos
+  - noconfigpkgs
+  - noconfigpkgs::git
+
+dependencies:
+  - baseusers
+  - packages_repos
+  - noconfigpkgs
+  - apt
+  - yum
+  - puppi
+  - stdlib
+
+
+# install packages
+noconfigpkgs::default_packages:
+  RedHat:
+    - 'passwd'
+    - 'acl'
+    - 'diffutils'
+    - 'traceroute'
+    - 'sysstat'
+    - 'mailx'
+    - 'mtr'
+    - 'vim-enhanced'
+    - 'screen'
+    - 'wget'
+    - 'curl'
+    - 'unzip'
+    - 'tar'
+    - 'cyrus-sasl-plain'
+    - 'zsh'
+  Debian:
+    - 'zsh'
+    - 'sysstat'
+    - 'atop'
+    - 'at'
+    - 'vim'
+    - 'screen'
+    - 'wget'
+    - 'curl'
+    - 'unzip'
+    - 'tar'
+
+# Create users
+baseusers::users :
+  atlassian :
+    name : 'atlassian'
+    home : '/home/atlassian'
+    shell : '/bin/bash'
+    gid  : 1010
+    uid  : 1010
+    comment : 'atlassian User'
+    password : '*'
+
+# create groups
+baseusers::groups:
+  hosting :
+    name : 'atlassian'
+    gid  : 1010
+
+# create apt/yum repositories
+packages_repos::repos:
+  Debian:
+    bintraybintray-adaptavist-deb:
+      location: 'https://dl.bintray.com/adaptavist/ubuntu/'
+      repos: '/dists/ubuntu/main/binary-amd64/'
+      include_src: false
+      release: ''
+      key: '8756C4F765C9AC3CB6B85D62379CE192D401AB61'
+      key_content: |
+        -----BEGIN PGP PUBLIC KEY BLOCK-----
+        Version: GnuPG v1
+
+        mQINBFTi8JIBEACcN1ucQ1uCOZ1owTELQV/6i4q7NbYdJ5wf7yPYfEugSo3yfbo3
+        Pw/XEvlnpDZmT155sGNOkteZtZMdcm5XhFbdtquLlrkjAcUGatq5rAt3eLAlvU7u
+        CBCDJg3ZaqpZti5ti2TfiaXHeawTpxaTb3V5tT4NYhY0aJqe0MGoVl2yZyoKMWsL
+        8XcUiJkUYnpu98BvnzO9ORSnKWHk60YxzZuHh5buMNiV4aI331ogiTxqISzTwEdQ
+        ygtlp4IeqE6w4x4RUOqQg/mu0xhqnP375KksPtKALLEr9vgqsJXfWVa5UmNl+rZP
+        gMiNEt+Abwewa6IQGgSU8GuxMp3qHxZtJQRNwIPx/yb7FngtWrUKIoQXs9xJwdJB
+        z4vhfFVeQlyPkEycQNcRfHVzK62oF8L5Jj/D8BIGAD+dj3x10Cy+qVK6BTY/F1zv
+        5iL12LjSlz8DtmTbqjit0WGoULjXFZALAU36q6FmE/nMcFuLaTUIinGV4fMvLgf9
+        Zn44juAhZMweOt63Pn4n/K0W+uOdrLSmGxJDhoxztabUdIpIMsw44wZ8gnSmPAef
+        IDTCjJO2x9s2YuaZbgstpJldooxGJ+FTe52QXFphti+tkiGOg6Tpj8Xq3+ZEM3L9
+        Js38SSdys0XBCHYiCv3/4Fk4jspTsCFrDzJ9HqNjsiktxPm9szmUZ72RjwARAQAB
+        tChCaW50cmF5IChieSBKRnJvZykgPGJpbnRyYXlAYmludHJheS5jb20+iQI4BBMB
+        AgAiBQJU4vCSAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRA3nOGS1AGr
+        YTe6D/9lwml8fFJxfF2dI8GNPMmRAwnewu85JSWE/Yc3adlWL+NqXhUotDbSgUXl
+        RmC22fxBFaWipiCMjDm5R+dthOFmaBnnIdWmTvrTyupJlsYHCj1FN/5izgYpband
+        qFYbpdX34fOiH+kFVKOQI5WlMGvgYRTusk5pfORK87/e9zXFFuuc4OmgKgW0JX3c
+        faFp8HnJFVl6j7us384U/m06BBUbJb/az7IZNZXu0FPfL9jUIcWbGRWjmIdySE9b
+        yMOB95QPNlTrnGcjVuWa1gTN5uEbMRa5sVq6SAxmph5eGspJrJ05Bjwk5rS3LkLE
+        1tv31Bpeb+2jIoIXUJj8ESS/6bLK6/d7TbjMrdcRvSIZggf1u0JnjnsT6eYmfY1m
+        iVhKy4FFTyofDOlyt1k7lEYH+iJ4Z5ij/b6wpoUViKv+zqDRrSSbwun111f8rH7W
+        WldC3rEsH5R8J+jm54P5pwC/LnBg53GvofpntARLNUPvcFVp7Hjue3kbTVx51pxx
+        BBf593UnAXs+pZMyhl/synSngjpebufQHPeX1jJyGdXkDnavEp8M7yqf61zj8+sj
+        dFPP4Sdf3sv35zJmals9L33Bjsmhvs5LtNFDJQDea/NVGcgfMHzwrMJ9GcfVPkLk
+        31c0+OaK11hkDZFZYrBWU6FWsj8lICJPHlmFsU/zirfkvFYJ3rkCDQRU4vCSARAA
+        qvnUkerHq1Fq3ptYrYsNDLJSLbBch7jldPivGVDi0YHv2qwUnxo5O2GTxcyDFW8V
+        6Oy2InIhwsnWfSux3agqsoAuJNiFfvOS5dO2X62jx2tr34F7IbtN/lWXDHKeicbP
+        lD5VR1e0hNkd6NsPiryqsyy0S2+mgURKCQrCOtB01sj47B4h62iflxTZdC09trSD
+        yRYzk3lSlP/DjAbNzuapd84HTBtwxRgEtgC4gm9cIfmICfXPEwOOEediadM9V1GF
+        71dvfBcxw+p+3o8In9jDVJCxe6BX0pJ0C5AMNVrqpMGJ90GKHH6fGlubt9d/b1lk
+        eVdsi1nhiNfv7KUyaj/HlwZxfoz1rooPxpBxq1gp/jE+17/E09sEeK3YXrZGD5zz
+        V9K2vo1EWW4nurTvwuTlk2I7q00swQ4j8TS3McVDY6zjMyG3Cy4UkUNA0xS4gueg
+        /uVLzyFGPxol+Tu8eIhdZMEj3KF89cPsc8wsHxWYPaBOb6BwMm6xpExQiG+TqPli
+        lgwmOeiu8hyyFE+FJohdi4ms+4HrE3OchUhSYT9FqZFV+hcQ7qAq8kMdC9/Kg/uH
+        OOOTe2lH1ZqmzgQaeDkaSf8NLPEW/eOskPE01AdOqLaL8iM9YmbLo9MlPZM2WKL6
+        2aSiS3gxGNk4cXVPzt2ZAKMBHk41visnXU0/a1LoIAMAEQEAAYkCHwQYAQIACQUC
+        VOLwkgIbDAAKCRA3nOGS1AGrYcySEACZIe/xvLjEPhiVtUqcACPyXL4U7uA+V5Ob
+        ZVRmKKlkuoq3AQGQs/LAyCSYIGRw13hAn1X6tnireTv+vEoMDaX0sB1qUw49WOuB
+        8h71NaF/UYaPehjRWyNNq5Ul+icNwc8I8tgfkUUFCm/a5nJh8pZWfo+404ujEJzI
+        I2Qk6SoZqhbq2xrTgCrrKHxG5Gp+a35Y2v+TC8OkAN3Gu9LBg39t058xArBikk8I
+        jneCbIpDV5Fv5O9J1GuFEHFH2NIolaGppEOswd0ALs3zOmQ8KOZxLa4Gnn59gkQ6
+        /8Db1zXTW1QUQWiylvFte0q+fcSwhKEgJKyyN0ptk4Y27rclZxLMvPAjW19bqnVR
+        tigjWHJlxmBzX2bodLWbx1eRiS5QIeOk32CZlQN7EE0lniKLVNHReCrBmiBVRH9k
+        sKFbFafs2sI97FP2QySQuugcM30qDutA2Coo58SoAYAYM+0JlKSwwFRH0mGDPCiw
+        xSzOu4BNlIoxQh3EzrsmiyiB4hWPn9qzX5VM2IXvtL1Wzv8rUtpANkso9MPjsMAf
+        1Y/KBBaUm0QehoMwCWF/1KwsF9ENu6xon4l+GfkPhuCsEHEdqWIVGXrDLSshMGZ7
+        HdyAtUHPXXFV0FCT3KqV4UiJrjAzv7jqfSSUsXT8Qf4H+hC8lTfSBbFNfxP14T+E
+        JESa2SNRfw==
+        =EI0Z
+        -----END PGP PUBLIC KEY BLOCK-----
+  RedHat:
+    bintraybintray-adaptavist-rpm:
+      baseurl: 'https://dl.bintray.com/adaptavist/rpm'
+      descr: "adaptavist avst-app repository"
+      enabled: 1
+      gpgcheck: 0
+      priority: 1

data/config/deploy.rb

@@ -0,0 +1,79 @@
+
+lock '3.2.1'
+
+set :application, 'my_app_name'
+set :repo_url, ENV["cap_git_repo_url"]
+set :branch, ENV["cap_reference_name"] || ENV["cap_branch_name"] || "master" 
+
+# Default branch is :master
+# ask :branch, proc { `git rev-parse --abbrev-ref HEAD`.chomp }.call
+
+# Default deploy_to directory is /var/www/my_app
+set :deploy_to, ENV["destination_folder"]
+
+# Default value for :scm is :git
+set :scm, :git
+
+# Default value for :format is :pretty
+set :format, :pretty
+
+# Default value for :log_level is :debug
+# set :log_level, :debug
+
+# Default value for :pty is false
+set :pty, true
+
+# Default value for :linked_files is []
+# set :linked_files, %w{config/database.yml}
+
+# Default value for linked_dirs is []
+# set :linked_dirs, %w{bin log tmp/pids tmp/cache tmp/sockets vendor/bundle public/system}
+# set :use_sudo, true
+# set :pty, true
+# default_run_options[:pty] = true
+
+set :ssh_options, {
+  forward_agent: false,
+  keys: [ENV["cap_access_password"]]
+}
+
+server ENV["cap_ip_address"], user: ENV["cap_access_user"], port: 22, password: ENV["cap_access_password"], roles: %w{app}
+
+# Default value for default_env is {}
+# set :default_env, { path: "/usr/local/rvm/rubies/ruby-2.1.2/bin/:/usr/local/rvm/gems/ruby-2.1.2/bin/:$PATH" }
+
+# Default value for keep_releases is 5
+set :keep_releases, 50000
+
+after 'deploy', 'prepare_puppet_and_execute'
+
+set :default_env, {
+  'PATH' => "/usr/local/rvm/gems/ruby-2.0.0-p481/bin:/usr/local/rvm/gems/ruby-2.0.0-p481@global/bin:/usr/local/rvm/rubies/ruby-2.0.0-p481/bin:/usr/local/rvm/bin:$PATH",
+  'RUBY_VERSION' => '2.0.0',
+  'GEM_HOME'     => '/usr/local/rvm/gems/ruby-2.0.0-p481',
+  'GEM_PATH'     => '/usr/local/rvm/gems/ruby-2.0.0-p481',
+  # 'BUNDLE_PATH'  => '/path/to/.rvm/gems/ree-1.8.7-2010.01'  # If you are using bundler.
+}
+
+namespace :deploy do
+
+  desc 'Restart application'
+  task :restart do
+    on roles(:app), in: :sequence, wait: 5 do
+      # Your restart mechanism here, for example:
+      # execute :touch, release_path.join('tmp/restart.txt')
+    end
+  end
+
+  after :publishing, :restart
+
+  after :restart, :clear_cache do
+    on roles(:web), in: :groups, limit: 3, wait: 10 do
+      # Here we can do anything such as:
+      # within release_path do
+      #   execute :rake, 'cache:clear'
+      # end
+    end
+  end
+
+end

data/config/deploy/production.rb

@@ -0,0 +1,48 @@
+# Simple Role Syntax
+# ==================
+# Supports bulk-adding hosts to roles, the primary server in each group
+# is considered to be the first unless any hosts have the primary
+# property set.  Don't declare `role :all`, it's a meta role.
+
+# role :app, %w{example.com}
+
+
+# Extended Server Syntax
+# ======================
+# This can be used to drop a more detailed server definition into the
+# server list. The second argument is a, or duck-types, Hash and is
+# used to set extended properties on the server.
+
+# server '178.250.52.77', user: 'root', roles: %w{app}, my_property: :my_value
+# set :password, ask('Server password', nil)
+# # fetch(:password)
+# server '178.250.53.87', user: 'root', port: 22, password: '***REMOVED***', roles: %w{app}
+
+# set :rvm_type, :system                     # Defaults to: :auto
+# set :rvm_ruby_version, '2.1.2p95'      # Defaults to: 'default'
+
+# Custom SSH Options
+# ==================
+# You may pass any option but keep in mind that net/ssh understands a
+# limited set of options, consult[net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start).
+#
+# Global options
+# --------------
+#  set :ssh_options, {
+#    keys: %w(/home/rlisowski/.ssh/id_rsa),
+#    forward_agent: false,
+#    auth_methods: %w(password)
+#  }
+#
+# And/or per server (overrides global)
+# ------------------------------------
+# server 'example.com',
+#   user: 'user_name',
+#   roles: %w{web app},
+#   ssh_options: {
+#     user: 'user_name', # overrides user setting above
+#     keys: %w(/home/user_name/.ssh/id_rsa),
+#     forward_agent: false,
+#     auth_methods: %w(publickey password)
+#     # password: 'please use keys'
+#   }

data/lib/avst-cloud.rb

@@ -0,0 +1,28 @@
+# Copyright 2015 Adaptavist.com Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative './avst-cloud/aws_connection.rb'
+require_relative './avst-cloud/aws_server.rb'
+require_relative './avst-cloud/rackspace_connection.rb'
+require_relative './avst-cloud/rackspace_server.rb'
+require_relative './avst-cloud/logging.rb'
+require 'fog'
+require 'net/ssh'
+require 'net/scp'
+require 'uri'
+require 'colorize'
+
+module AvstCloud
+
+end

data/lib/avst-cloud/aws_connection.rb

@@ -0,0 +1,203 @@
+# Copyright 2015 Adaptavist.com Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative './cloud_connection.rb'
+
+module AvstCloud
+    
+    class AwsConnection < AvstCloud::CloudConnection
+        
+        attr_accessor :region
+
+        def initialize(provider_user, provider_pass, region)
+            super('aws', provider_user, provider_pass)
+            @region = region
+        end
+        
+        def server(server_name, root_user, root_password, os=nil)
+            server = find_fog_server(server_name)
+            if !root_user and os
+                root_user = user_from_os(os)
+            end
+            AvstCloud::AwsServer.new(server, server_name, server.public_ip_address, root_user, root_password)
+        end
+
+        def create_server(server_name, flavour, os, key_name, ssh_key, subnet_id, security_group_ids, ebs_size, hdd_device_path, ami_image_id, availability_zone)
+
+            # Permit named instances from DEFAULT_FLAVOURS
+            flavour = flavour || "t2.micro"
+            os = os || "ubuntu-14"
+            ami_image_id = ami_image_id || "ami-f0b11187"
+            device_name = hdd_device_path || '/dev/sda1'
+
+            root_user = user_from_os(os)
+            unless File.file?(ssh_key)
+                logger.error "Could not find local SSH key '#{ssh_key}'".red
+                raise "Could not find local SSH key '#{ssh_key}'"
+            end
+
+            existing_servers    = all_named_servers(server_name)
+            restartable_servers = existing_servers.select{ |serv| serv.state == 'stopped' }
+            running_servers     = existing_servers.select{ |serv| serv.state != 'stopped' && serv.state != 'terminated' }
+
+            if running_servers.length > 0
+                running_servers.each do |server|
+                    logger.error "Server #{server_name} with id #{server.id} found in state: #{server.state}".yellow
+                end
+                raise "Server with the same name found!"
+
+            elsif restartable_servers.length > 0
+                if restartable_servers.length > 1
+                    running_servers.each do |server|
+                        logger.error "Server #{server_name} with id #{server.id} found in state: #{server.state}. Can not restart".yellow
+                    end
+                    raise "Too many servers can be restarted."
+                end
+                server = restartable_servers.first
+                server.start
+                result_server = AvstCloud::AwsServer.new(server, server_name, server.public_ip_address, root_user, ssh_key)
+                result_server.wait_for_state() {|serv| serv.ready?}
+                logger.debug "[DONE]\n\n"
+                logger.debug "The server was successfully re-started.\n\n"
+                result_server
+            else
+                logger.debug "Creating EC2 server:"
+                logger.debug "Server name        - #{server_name}"
+                logger.debug "Operating system   - #{os}"
+                logger.debug "image_template_id  - #{ami_image_id}"
+                logger.debug "flavour            - #{flavour}"
+                logger.debug "key_name           - #{key_name}"
+                logger.debug "ssh_key            - #{ssh_key}"
+                logger.debug "subnet_id          - #{subnet_id}"
+                logger.debug "security_group_ids - #{security_group_ids}"
+                logger.debug "region             - #{@region}"
+                logger.debug "availability_zone  - #{availability_zone}"
+                logger.debug "ebs_size           - #{ebs_size}"
+                logger.debug "hdd_device_path    - #{device_name}"
+
+                create_ebs_volume = nil
+                if ebs_size
+                    # in case of centos ami we need to use /dev/xvda this is ami dependent
+                    create_ebs_volume = [ 
+                                            { 
+                                                :DeviceName => device_name,
+                                                'Ebs.VolumeType' => 'gp2',
+                                                'Ebs.VolumeSize' => ebs_size,
+                                            } 
+                                        ]
+                end
+
+                # create server
+                server = connect.servers.create :tags => {
+                                                    'Name' => server_name,
+                                                    'os' => os
+                                                },
+                                                :flavor_id => flavour,
+                                                :image_id => ami_image_id,
+                                                :key_name => key_name,
+                                                :subnet_id => subnet_id,
+                                                :associate_public_ip => true,
+                                                :security_group_ids => security_group_ids,
+                                                :availability_zone => availability_zone,
+                                                :block_device_mapping => create_ebs_volume
+                
+                result_server = AvstCloud::AwsServer.new(server, server_name, nil, root_user, ssh_key)
+                # result_server.logger = logger
+                # Check every 5 seconds to see if server is in the active state (ready?).
+                # If the server has not been built in 5 minutes (600 seconds) an exception will be raised.
+                result_server.wait_for_state() {|serv| serv.ready?}
+
+                logger.debug "[DONE]\n\n"
+
+                logger.debug "The server has been successfully created, to login onto the server:\n"
+                logger.debug "\t ssh -i #{ssh_key} #{root_user}@#{server.public_ip_address}\n"
+            
+                result_server.ip_address = server.public_ip_address
+                result_server
+            end
+        end
+
+        def server_status(server_name)
+            servers = all_named_servers(server_name).select{|serv| serv.state != 'terminated'}
+            if servers.length > 0
+                servers.each do |server|
+                    logger.debug "Server #{server.id} with name '#{server_name}' exists and has state: #{server.state}"
+                end
+            else
+                logger.debug "Server not found for name: #{server_name}"
+            end
+        end
+
+        def list_flavours
+            connect.flavors.each do |fl|
+                logger.debug fl.inspect
+            end
+        end
+
+        def list_images
+            connect.images.each do |im|
+                logger.debug im.inspect
+            end
+        end
+        
+        # Returns list of servers from fog
+        def list_known_servers
+            connect.servers.all
+        end
+
+        def find_fog_server(server_name, should_fail=true)
+            servers = all_named_servers(server_name).select{|serv| serv.state != 'terminated'}
+            unless servers.length == 1    
+                logger.debug "Found #{servers.length} servers for name: #{server_name}".yellow
+                if should_fail
+                    raise "Found #{servers.length} servers for name: #{server_name}"
+                end
+            end
+            servers.first
+        end
+
+    private
+        def user_from_os(os)
+            case os.to_s
+                when /^ubuntu/
+                    user = "ubuntu"
+                when /^debian/
+                    user = "admin"
+                when /^centos/
+                    user = "ec2-user"
+                when /^redhat/
+                    user = "ec2-user"
+                else
+                    user = "root"
+            end
+            user
+        end
+        def connect
+            unless @connection
+                logger.debug "Creating new connection to AWS"
+                @connection = Fog::Compute.new({
+                    :provider               => 'AWS',
+                    :region                 => @region,
+                    :aws_access_key_id      => @provider_access_user,
+                    :aws_secret_access_key  => @provider_access_pass
+                })
+            end
+            @connection
+        end
+        
+        def all_named_servers(server_name)
+            connect.servers.all({'tag:Name' => server_name})
+        end
+    end
+end