pvcglue 0.1.5

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTRmZThlNzQ0YTkxNWI2Njg4NGNjZmUwMzdjN2VkMDMyYzdiYWVhZA==
+  data.tar.gz: !binary |-
+    Mjg3ODk4M2ZmZTc4NzAwY2NhNTYyYTgyZGJjMGMzNzI2YzBjMDgzYw==
+SHA512:
+  metadata.gz: !binary |-
+    N2RiMTUyZDhiMTlkZmM5OWM4ODRlMDBiOTM0MmQ1NDY5MDQ4NjRlZmU4NDU2
+    MTE5MmY5OTU4MzhkNGZiMmE5MTY4MzhkZDEzMTMyMmY5MTFkMGJjZTgyNWMx
+    Nzc2Y2E2YmRmNjNjYzJjYjA2YjFmNDY5NDc3M2NjMDE5OTMwOWU=
+  data.tar.gz: !binary |-
+    Zjk0YWNkYjE3YmYyZmYxZjFiMDBjYWIwZDdmMDk4ZmNiOTBjYWM0MjViNDM4
+    ODYxYTc5MjdmOTAxNGQ3OTkxOTBhOGYxYWI4M2FkNmM1MTg4ODAzNWYyMTc0
+    OGE4MzE2OGQ5MWNhZTcxMTNmOGNkZDg5MmI0MTdjYTI4MGZmNzQ=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in pvcglue.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Andrew Lyric
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# Basic Usage
+
+    pvc manager init # only once per machine
+    pvc manger build # only once
+    # create or edit configuration
+    pvc manager push # save the config
+    pvc <stage> build # anytime the config is changed
+    pvc <stage> env set KEY=value[, KEY2=value]
+    pvc <stage> capify
+    pvc <stage> deploy
+
+# Pvcglue
+
+    Commands:
+      pvc bootstrap -s, --stage=STAGE               # bootstrap...
+      pvc build -s, --stage=STAGE                   # build...
+      pvc c -s, --stage=STAGE                       # shortcut for console
+      pvc capify -s, --stage=STAGE                  # update capistrano configuration
+      pvc console -s, --stage=STAGE                 # open rails console
+      pvc db SUBCOMMAND ...ARGS                     # db utils
+      pvc deploy -s, --stage=STAGE                  # deploy the app
+      pvc env SUBCOMMAND ...ARGS -s, --stage=STAGE  # manage stage environment
+      pvc help [COMMAND]                            # Describe available commands or one specific c...
+      pvc info                                      # show the pvcglue version and cloud settings
+      pvc m -s, --stage=STAGE                       # enable or disable maintenance mode
+      pvc maint -s, --stage=STAGE                   # enable or disable maintenance mode
+      pvc maintenance -s, --stage=STAGE             # enable or disable maintenance mode
+      pvc manager SUBCOMMAND ...ARGS                # manage manager
+      pvc s -s, --stage=STAGE                       # shell
+      pvc sh -s, --stage=STAGE                      # run interactive shell on node
+      pvc ssl SUBCOMMAND ...ARGS -s, --stage=STAGE  # manage ssl certificates
+      pvc version                                   # show the version of PVC...
+
+https://github.com/radar/guides/blob/master/gem-development.md
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    group :development do
+      gem 'pvcglue', "~> 0.1.0", :github => 'talyric/pvcglue', :branch => 'master', :require => false
+
+      # This should be used once gem is 'official' :)
+      #gem 'pvcglue'
+    end
+
+    gem 'pvcglue_dbutils', "~> 0.5.1", :github => 'talyric/pvcglue_dbutils', :branch => 'master' # must be available in all environments
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pvcglue
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+License
+-------
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/pvc

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+require 'pvcglue/cli'
+
+# Allow use of Capistrano style environment syntax and convert to 'standard' syntax
+# Example:  `pvc production bootstrap` ==> `pvc bootstrap --stage=production`
+# TODO: refactor to use a list of user specified environments
+if ARGV.count >= 2 && %w[alpha beta gamma delta preview production staging all].include?(ARGV[0])
+  ARGV[0], ARGV[1] = ARGV[1], "--stage=#{ARGV[0]}"
+end
+
+# puts ARGV.inspect
+
+Pvcglue::CLI.start

data/lib/pvcglue.rb

@@ -0,0 +1,43 @@
+require "pvcglue/version"
+require "thor"
+require "pvcglue/configuration"
+require "pvcglue/manager"
+require "pvcglue/cloud"
+require "pvcglue/packages"
+require "pvcglue/bootstrap"
+require "pvcglue/nodes"
+require "pvcglue/env"
+require "pvcglue/deploy"
+require "pvcglue/capistrano"
+require "pvcglue/ssl"
+require "pvcglue/db"
+require "pvcglue/toml_pvc_dumper.rb"
+require "tilt"
+
+# puts File.join(File.dirname(__FILE__), 'pvcglue', 'packages', '*.rb')
+
+module Pvcglue
+
+  def self.gem_dir
+    Gem::Specification.find_by_name('pvcglue').gem_dir
+  end
+
+  def self.template_file_name(template)
+    File.join(Pvcglue::gem_dir, 'lib', 'pvcglue', 'templates', template)
+  end
+
+  def self.render_template(template, file_name = nil)
+    data = Tilt.new(Pvcglue.template_file_name(template)).render
+    if file_name
+      File.write(file_name, data)
+    end
+    data
+  end
+
+  class Version
+    def self.version
+      VERSION
+    end
+  end
+
+end

data/lib/pvcglue/all_the_things.rb

@@ -0,0 +1,7 @@
+require "orca/extensions/apt"
+require "orca/extensions/file_sync"
+#/home/andrew/projects/pvcglue/lib/pvcglue/packages
+#/home/andrew/projects/pvcglue/lib/pvcglue/packages/bootstrap_manager.rb
+#/home/andrew/projects/pvcglue
+#puts Pvcglue::gem_dir
+#Dir[File.join(Pvcglue::gem_dir,'lib', 'pvcglue', 'packages', '*.rb')].each { |file| puts "#{file}*********************************************************"; require file }

data/lib/pvcglue/bootstrap.rb

@@ -0,0 +1,8 @@
+module Pvcglue
+  class Bootstrap
+    def self.run(roles)
+      # puts "This is where it should bootstrap #{Pvcglue.cloud.stage_name}.  :)"
+      Pvcglue::Packages.apply('bootstrap'.to_sym, :bootstrap, Pvcglue.cloud.nodes_in_stage(roles), 'root')
+    end
+  end
+end

data/lib/pvcglue/capistrano.rb

@@ -0,0 +1,35 @@
+module Pvcglue
+  class Capistrano
+    # TODO:  Add requirements to gem file:  capistrano-rails, etc.
+    def self.capify
+      Pvcglue.render_template('capfile.erb', capfile_file_name)
+      Pvcglue.render_template('deploy.rb.erb', common_deploy_file_name)
+      Pvcglue.render_template('stage-deploy.rb.erb', stage_deploy_file_name)
+    end
+
+    def self.capfile_file_name
+      File.join(Pvcglue.configuration.application_dir, 'Capfile')
+    end
+
+    def self.application_config_dir
+      File.join(Pvcglue.configuration.application_dir, 'config')
+    end
+
+    def self.common_deploy_file_name
+      File.join(application_config_dir, 'deploy.rb')
+    end
+
+    def self.stage_deploy_dir
+      File.join(application_config_dir, 'deploy')
+    end
+
+    def self.stage_deploy_file_name
+      `mkdir -p #{stage_deploy_dir}`
+      File.join(stage_deploy_dir, "#{Pvcglue.cloud.stage_name_validated}.rb")
+    end
+
+    def self.deploy
+      system("cap #{Pvcglue.cloud.stage_name} deploy")
+    end
+  end
+end

data/lib/pvcglue/cli.rb

@@ -0,0 +1,150 @@
+require 'thor'
+require 'orca'
+require 'pvcglue'
+
+module Pvcglue
+
+  class CLI < Thor
+
+    def initialize(args = [], local_options = {}, config = {})
+      super
+      Pvcglue.cloud.set_stage(options[:stage])
+      # puts "/\\"*80
+      # puts options.inspect
+      # puts "*"*80
+    end
+
+    desc "version", "show the version of PVC..."
+
+    def version
+      puts Pvcglue::Version.version
+    end
+
+    desc "info", "show the pvcglue version and cloud settings"
+
+    def info
+      puts "Pvcglue version #{Pvcglue::Version.version}"
+      puts "  Manager settings:"
+      Pvcglue.configuration.options.each { |k, v| puts "    #{k}=#{v}" }
+    end
+
+    desc "show", "show the pvcglue version and cloud settings"
+
+    def show
+      info
+    end
+
+    desc "bootstrap", "bootstrap..."
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def bootstrap(roles = 'all')
+      Pvcglue::Bootstrap.run(roles)
+    end
+
+    desc "build", "build..."
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def build(roles = 'all')
+      Pvcglue::Nodes.build(roles)
+    end
+
+    desc "console", "open rails console"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def console(server='web')
+      node = Pvcglue.cloud.find_node(server)
+      node_name = node.keys.first
+      node_data = node.values.first
+      # puts "*"*80
+      # puts node.inspect
+      puts "Connection to #{node_name} (#{node_data[:public_ip]}) as user 'deploy'..."
+      working_dir = Pvcglue.cloud.deploy_to_app_current_dir
+      system(%(ssh -t deploy@#{node_data[:public_ip]} "cd #{working_dir} && echo 'Starting #{options[:stage].upcase} Rails console in #{working_dir}' && RAILS_ENV=#{options[:stage].downcase} script/rails c"))
+
+    end
+
+    desc "c", "shortcut for console"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def c(server='web')
+      console(server)
+    end
+
+    desc "manager SUBCOMMAND ...ARGS", "manage manager"
+    #banner 'manager'
+    subcommand "manager", Manager
+
+    desc "env SUBCOMMAND ...ARGS", "manage stage environment"
+    method_option :stage, :required => true, :aliases => "-s"
+    #banner 'manager'
+    subcommand "env", Env
+
+    desc "db SUBCOMMAND ...ARGS", "db utils"
+    method_option :stage, :aliases => "-s"
+    subcommand "db", Db
+
+    desc "ssl SUBCOMMAND ...ARGS", "manage ssl certificates"
+    method_option :stage, :required => true, :aliases => "-s"
+    subcommand "ssl", Ssl
+
+
+    desc "maintenance", "enable or disable maintenance mode"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def maintenance(mode)
+      raise(Thor::Error, "invalid maintenance mode :(  (Hint:  try on or off.)") unless mode.in?(%w(on off))
+      Pvcglue.cloud.maintenance_mode = mode
+      Pvcglue::Packages.apply(:maintenance_mode, :maintenance, Pvcglue.cloud.nodes_in_stage('lb'))
+    end
+
+    desc "maint", "enable or disable maintenance mode"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def maint(mode)
+      maintenance(mode)
+    end
+
+    desc "m", "enable or disable maintenance mode"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def m(mode)
+      maintenance(mode)
+    end
+
+    desc "sh", "run interactive shell on node"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def sh(server='web') # `shell` is a Thor reserved word
+      node = Pvcglue.cloud.find_node(server)
+      node_name = node.keys.first
+      node_data = node.values.first
+      # puts "*"*80
+      # puts node.inspect
+      puts "Connection to #{node_name} (#{node_data[:public_ip]}) as user 'deploy'..."
+      system("ssh -p #{Pvcglue.cloud.port_in_context(:shell)} deploy@#{node_data[:public_ip]}")
+    end
+
+    desc "s", "shell"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def s(server='web')
+      sh(server)
+    end
+
+    desc "pvcify", "update capistrano, database.yml and other configurations"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def pvcify
+      Pvcglue::Capistrano.capify
+      Pvcglue::Db.configure_database_yml
+    end
+
+    desc "deploy", "deploy the app"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def deploy
+      Pvcglue::Capistrano.deploy
+    end
+  end
+
+end

data/lib/pvcglue/cloud.rb

@@ -0,0 +1,278 @@
+require "active_support/core_ext" # for `with_indifferent_access`
+
+
+module Pvcglue
+  class Cloud
+    attr_accessor :data
+    attr_accessor :current_node
+    attr_accessor :current_hostname
+    attr_accessor :maintenance_mode
+    attr_accessor :stage_env
+    attr_accessor :passenger_ruby
+    attr_accessor :port_in_node_context
+
+    def data
+      ::Pvcglue::Manager.initialize_cloud_data unless @data
+      @data
+    end
+
+    def data=(data)
+      @data = data.with_indifferent_access # We may not want this dependency.
+    end
+
+    def current_node
+      raise "Current node not set." if @current_node.nil?
+      @current_node
+    end
+
+    def current_hostname
+      raise "Current current_hostname not set." if @current_hostname.nil?
+      @current_hostname
+    end
+
+    def set_stage(stage)
+      @stage_name = stage
+      @stage_name.downcase! if @stage_name
+    end
+
+    def stage_name
+      @stage_name
+    end
+
+    def stage_name_validated
+      # TODO:  Document better or fix root cause
+      # Work-around for orca file packages that are loaded when required, but stage_name is not going to be used
+      # raise "stage_name is required in this context" unless @stage_name
+      @stage_name
+    end
+
+    def stage
+      # puts data[app_name].inspect
+      # puts data[app_name][:stages].inspect
+      # puts data[app_name][:stages][stage_name].inspect
+      data[app_name][:stages][stage_name]
+    end
+
+    def stage_roles
+      raise(Thor::Error, "Stage not defined:  #{stage_name}.") if stage.nil?
+      stage[:roles]
+    end
+
+    def local_file_name
+      File.join(application_dir, Pvcglue::Manager.cloud_data_file_name_base)
+    end
+
+    def env_local_file_name
+      File.join(application_dir, Pvcglue::Env.stage_env_file_name_base)
+    end
+
+    def application_dir
+      Pvcglue.configuration.application_dir
+    end
+
+    # find node by full node_name or by matching prefix of node_name
+    def find_node(node_name)
+      puts "*"*80
+      raise(Thor::Error, "Node not specified.") if node_name.nil? || node_name.empty?
+      return {node_name => nodes_in_stage[node_name]} if nodes_in_stage[node_name]
+      puts "-"*80
+      nodes_in_stage.each do |key, value|
+        puts key
+        return {key => value} if key.start_with?(node_name)
+      end
+      raise(Thor::Error, "Not found:  #{node_name} in #{stage_name}.")
+    end
+
+    def nodes_in_stage(role_filter = 'all')
+      # puts (stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }).inspect
+      # stage_roles.values.each_with_object({}) { |node, nodes| nodes.merge!(node) }
+      nodes = stage_roles.each_with_object({}) do |(role, node), nodes|
+        if role_filter == 'all' || role == role_filter
+          nodes.merge!(node)
+        end
+      end
+      # puts nodes.inspect
+      # puts "nodes_in_stage: only first returned"+"!*"*80
+      # out = {}
+      # out["memcached"] = nodes["memcached"]
+      # puts out.inspect
+      # out
+    end
+
+    # ENV['PVC_DEPLOY_TO_BASE'] = stage_data[:deploy_to] || '/sites'
+    def deploy_to_base_dir
+      # stage[:deploy_to] || '/sites' # TODO:  verify if server setup supports `:deploy_to` override
+      Pvcglue.configuration.web_app_base_dir # TODO:  server setup does not yet support `:deploy_to` override, and would have to be refactored at a higher level than stage.
+    end
+
+    # ENV['PVC_DEPLOY_TO_APP'] = "#{ENV['PVC_DEPLOY_TO_BASE']}/#{ENV['PVC_APP_NAME']}/#{ENV['PVC_STAGE']}"
+    def deploy_to_app_dir
+      File.join(deploy_to_base_dir, app_name, stage_name)
+    end
+
+    def deploy_to_app_current_dir
+      File.join(deploy_to_app_dir, 'current')
+    end
+
+    def deploy_to_app_shared_dir
+      File.join(deploy_to_app_dir, 'shared')
+    end
+
+    def env_file_name
+      File.join(deploy_to_app_shared_dir, ".env.#{stage_name_validated}")
+    end
+
+    def deploy_to_app_current_public_dir
+      File.join(deploy_to_app_current_dir, 'public')
+    end
+
+    def maintenance_files_dir
+      File.join(deploy_to_app_dir, 'maintenance')
+    end
+
+    def maintenance_mode_file_name
+      File.join(maintenance_files_dir, 'maintenance.on')
+    end
+
+    def deploy_to_app_current_temp_dir
+      File.join(deploy_to_app_current_dir, 'tmp')
+    end
+
+    def restart_txt_file_name
+      File.join(deploy_to_app_current_temp_dir, 'restart.txt')
+    end
+
+    def app_name
+      Pvcglue.configuration.application_name
+    end
+
+    def authorized_keys
+      data[app_name][:authorized_keys]
+    end
+
+    def ssh_ports
+      ports = []
+      from_all = data[app_name][:ssh_allowed_from_all_port].to_i
+      ports << from_all if from_all > 0
+      ports
+    end
+
+    def timezone
+      data[app_name][:time_zone] || 'America/Los_Angeles'
+    end
+
+    def firewall_allow_incoming_on_port
+      # These ports allow incoming connections from any ip address
+      ports = []
+      from_all = data[app_name][:ssh_allowed_from_all_port].to_i
+      ports << from_all if from_all > 0
+      ports.concat [80, 443] if current_node.values.first[:allow_public_access]
+      ports
+    end
+
+    def firewall_allow_incoming_from_ip
+      # Incoming connections to any port are allowed from these ip addresses
+      addresses = dev_ip_addresses
+      addresses.concat(stage_internal_addresses)
+      # puts addresses.inspect
+      addresses
+    end
+
+    def dev_ip_addresses
+      data[app_name][:dev_ip_addresses].values.each_with_object([]) { |address, addresses| addresses << address }
+    end
+
+    def stage_internal_addresses
+      nodes_in_stage.values.each_with_object([]) do |value, addresses|
+        addresses << value[:public_ip]
+        addresses << value[:private_ip] if value[:private_ip]
+      end
+    end
+
+    # app_stage_name = "#{ENV['PVC_APP_NAME']}_#{ENV['PVC_STAGE']}".downcase
+    def app_and_stage_name
+      "#{app_name}_#{stage_name}".downcase
+    end
+
+    def domains
+      stage[:domains]
+    end
+
+    def ssl_mode
+      stage[:ssl].to_sym || :none
+    end
+
+    def delayed_job_args
+      stage[:delayed_job_args]
+    end
+
+    def repo_url
+      data[app_name][:repo_url]
+    end
+
+    def swapfile_size
+      data[app_name][:swapfile_size] || stage[:swapfile_size] || 1024
+    end
+
+    def gems
+      data[app_name][:gems]
+    end
+
+    def db_rebuild
+      !!stage[:db_rebuild]
+    end
+
+    def nginx_config_path
+      '/etc/nginx'
+    end
+
+    def nginx_config_ssl_path
+      File.join(nginx_config_path, 'ssl')
+    end
+
+    def nginx_ssl_crt_file_name
+      File.join(nginx_config_ssl_path, "#{app_and_stage_name}.crt")
+    end
+
+    def nginx_ssl_key_file_name
+      File.join(nginx_config_ssl_path, "#{app_and_stage_name}.key")
+    end
+
+    def ssl_crt
+      stage[:ssl_crt]
+    end
+
+    def ssl_key
+      stage[:ssl_key]
+    end
+
+    def port_in_context(context)
+      case context
+        when :bootstrap, :manager
+          port = "22"
+        when :env, :build, :shell, :deploy
+          port = data[app_name][:ssh_allowed_from_all_port] || "22"
+        else
+          raise "Context not specified or invalid"
+      end
+      puts "Setting port to #{port}"
+      @port_in_node_context = port
+    end
+
+    def port_in_node_context
+      raise "Context not specified or invalid" if @port_in_node_context.nil?
+      puts "Setting port_in_node_context to #{@port_in_node_context}"
+      @port_in_node_context
+    end
+  end
+
+  def self.cloud
+    @cloud ||= Cloud.new
+  end
+
+  def self.cloud=(config)
+    @cloud = config
+  end
+
+
+end