pvcglue 0.9.3 → 0.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02dc01c9dfc836f04aac2b7e7d099dd0cad4dc11
-  data.tar.gz: 3b2ee3d8c9bb49fee054d5c4ddb889b90fd69ebf
+  metadata.gz: e8640111473b3ed136163c0bb0fb2a7af3aca750
+  data.tar.gz: c509475f64ca8e936b661d238840be07b1ba435c
 SHA512:
-  metadata.gz: 362b79a99c872b3aba193122ca001ee57f7942d742b53ea4fec293961cec5e8362362556fd7b63b802f889a325e5caf7a622715d20356df75fb227f258ec50fe
-  data.tar.gz: 9145f5653e913be83eb8b2165bc5479fafa8c84a905637c0e81b6fc378c00e0948e7806c8f161367f1d572863795f1e46a7c4e9a6c15b0691c97ab222966b1a2
+  metadata.gz: 6880a84309e9d85cb6cd8e4965620f35672e91e55a7ce614e79b97dbba5041070ce6b53bc19f7f9e6a7842d4af0e6bd7d76e119a07e8b90906bdc0e2be3c8955
+  data.tar.gz: 47f8cded36079c27e0db96acca25db1ebfaecb273a95bff1ee42c0e8ba5c61e79b7bef22cee4685c94dd538008059d7604d45ffda41298c6a20c4ec36d5eb404

data/bin/pvc

@@ -31,7 +31,11 @@ def capistrano_style_environment
       end
     else
       Pvcglue::CLI.start
+      Pvcglue.docs.done
     end
+  else
+    Pvcglue::CLI.start
+    Pvcglue.docs.done
   end
 end
 

data/lib/pvcglue.rb

@@ -5,6 +5,7 @@ require 'pvcglue/manager'
 require 'pvcglue/cloud'
 require 'pvcglue/packages'
 Dir[File.dirname(__FILE__) + '/pvcglue/packages/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/pvcglue/cloud_providers/*.rb'].each { |file| require file }
 require 'pvcglue/bootstrap'
 require 'pvcglue/nodes'
 require 'pvcglue/stack'
@@ -17,21 +18,45 @@ require 'pvcglue/toml_pvc_dumper'
 require 'pvcglue/local'
 require 'pvcglue/monit'
 require 'pvcglue/pvcify'
+require 'pvcglue/docs'
 require 'tilt'
 require 'awesome_print'
 require 'hashie'
 require 'pvcglue/custom_hashie'
 require 'pvcglue/minion'
+require 'pvcglue/cloud_providers'
 require 'droplet_kit'
-require 'pvcglue/digital_ocean'
+# require 'pvcglue/digital_ocean'
 require 'logger'
 require 'pvcglue/connection'
 require 'paint'
-require 'pry'
+# require 'pry'
+require 'net/http'
+require 'byebug'
+
+
+# require 'thor'
+# require 'tilt'
+# require 'awesome_print'
+# require 'hashie'
+# require 'pvcglue/custom_hashie'
+# require 'logger'
+# Dir[File.dirname(__FILE__) + '/pvcglue/*.rb'].each { |file| require file unless File.basename(file) == 'cli.rb' }
+# require 'pvcglue/cli'
+# Dir[File.dirname(__FILE__) + '/pvcglue/packages/*.rb'].each { |file| require file }
+# Dir[File.dirname(__FILE__) + '/pvcglue/cloud_providers/*.rb'].each { |file| require file }
+# require 'droplet_kit'
+# require 'paint'
+# require 'pry'
+# require 'net/http'
+# require 'byebug'
 
 # puts File.join(File.dirname(__FILE__), 'pvcglue', 'packages', '*.rb')
 # pvc manager bootstrap --cloud_manager_override=local_cloud.pvcglue.toml --save_before_upload=save --verbose
 
+
+# TODO:  Set up a maintenance mode page and command to allow the message to be changed without a redeploy, like if Amazon S3 goes down...
+#
 module Pvcglue
   # def self.reset!
   #   ap Pvcglue.instance_variables
@@ -101,19 +126,19 @@ module Pvcglue
       # background = 'black'
       background = nil
       case severity[0..0]
-        when 'E'
-          foreground = 'red'
-        when 'W'
-          # foreground = 'black'
-          # background = 'yellow'
-          foreground = 'yellow'
-        when 'I'
-          foreground = 'purple'
-        when 'D'
-          foreground = 'cyan'
-        else
-          foreground = 'black'
-          background = 'red'
+      when 'E'
+        foreground = 'red'
+      when 'W'
+        # foreground = 'black'
+        # background = 'yellow'
+        foreground = 'yellow'
+      when 'I'
+        foreground = 'purple'
+      when 'D'
+        foreground = 'cyan'
+      else
+        foreground = 'black'
+        background = 'red'
       end
       # case severity[0..0]
       #    when 'E'
@@ -135,6 +160,10 @@ module Pvcglue
   mattr_accessor :logger_package_description
   mattr_accessor :logger_current_minion
 
+  mattr_accessor :docs do
+    Pvcglue::Docs.new(!!ARGV.detect { |arg| arg.downcase == '--docs' })
+  end
+
   def self.verbose?
     return if @filtering_verbose
     if Pvcglue.command_line_options[:verbose]
@@ -160,12 +189,15 @@ module Pvcglue
   end
 
   def self.template_file_name(template)
+    override = File.join(Pvcglue.configuration.template_override_dir, template)
+    return override if File.exists?(override)
     File.join(Pvcglue::gem_dir, 'lib', 'pvcglue', 'templates', template)
   end
 
   def self.render_template(template, file_name = nil)
     # puts '-'*80
     # puts "---> render_template(template=#{template}, file_name=#{file_name}"
+    # Pvcglue.logger.debug { "render_template(template=#{template}, file_name=#{file_name}" }
     data = Tilt.new(Pvcglue.template_file_name(template)).render
     if file_name
       File.write(file_name, data)
@@ -183,6 +215,14 @@ module Pvcglue
     true
   end
 
+  def self.system_get_stdout(cmd)
+    Pvcglue.logger.debug { cmd }
+    result = `#{cmd}`
+    Pvcglue.verbose? { result }
+    Pvcglue.logger.debug { "exit_code=#{$?.to_i}" }
+    result
+  end
+
   class Version
     def self.version
       VERSION

data/lib/pvcglue/cli.rb

@@ -1,5 +1,4 @@
 require 'thor'
-require 'orca'
 require 'pvcglue'
 
 module Pvcglue
@@ -12,9 +11,10 @@ module Pvcglue
     class_option :force_cert
     class_option :provision_only
     class_option :quiet
-    class_option :info
+    # class_option :info
     class_option :debug
     class_option :verbose
+    class_option :docs
 
     def initialize(args = [], local_options = {}, config = {})
       super
@@ -28,6 +28,7 @@ module Pvcglue
     desc "version", "show the version of PVC..."
 
     def version
+      byebug
       puts Pvcglue::Version.version
     end
 
@@ -261,6 +262,17 @@ module Pvcglue
       Pvcglue::Local.update_local_config_from_cache
     end
 
+    desc "render template", "debug use"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def render(template_file_name, machine_name = nil)
+      minion = machine_name.nil? ? nil : Pvcglue.cloud.find_minion_by_name(machine_name)
+      template = Tilt.new(Pvcglue.template_file_name(template_file_name))
+      data = template.render(self, minion: minion)
+      puts data
+      puts "---> Rendered '#{template_file_name}' from #{Pvcglue.template_file_name(template_file_name)}"
+    end
+
 
   end
 

data/lib/pvcglue/cloud.rb

@@ -15,7 +15,6 @@ module Pvcglue
     # attr_accessor :stage_secrets
 
 
-
     def data
       ::Pvcglue::Manager.initialize_cloud_data unless @data
       @data
@@ -335,9 +334,35 @@ module Pvcglue
       project[:gems] || {}
     end
 
+    def gem_file_data
+      @gem_file_data ||= begin
+        File.read(Pvcglue.configuration.gemfile_file_name)
+      end
+    end
+
+    def gem_installed?(gem)
+      value = gems[gem]
+      return value unless value == 'auto'
+      yield
+    end
+
     def whenever_gem_installed?
-      data = File.read(Pvcglue.configuration.gemfile_file_name)
-      data =~ /^\s*gem\s+['"]whenever['"]/
+      gem_installed?(:whenever) do
+        gem_file_data =~ /^\s*gem\s+['"]whenever['"]/
+      end
+    end
+
+    def delayed_job_gem_installed?
+      gem_installed?(:delayed_job) do
+        raise('Not implemented, yet.  :(')
+        gem_file_data =~ /^\s*gem\s+['"]whenever['"]/
+      end
+    end
+
+    def sidekiq_gem_installed?
+      gem_installed?(:sidekiq) do
+        gem_file_data =~ /^\s*gem\s+['"]sidekiq['"]/
+      end
     end
 
     def db_rebuild
@@ -370,12 +395,12 @@ module Pvcglue
 
     def port_in_context(context)
       case context
-        when :bootstrap, :manager
-          port = "22"
-        when :env, :build, :shell, :deploy, :maintenance
-          port = project[:ssh_allowed_from_all_port] || "22"
-        else
-          raise "Context not specified or invalid"
+      when :bootstrap, :manager
+        port = "22"
+      when :env, :build, :shell, :deploy, :maintenance
+        port = project[:ssh_allowed_from_all_port] || "22"
+      else
+        raise "Context not specified or invalid"
       end
       puts "Setting port to #{port}"
       @port_in_node_context = port
@@ -538,25 +563,16 @@ module Pvcglue
           minion.users = machine.users
           minion.cloud_id = machine.cloud_id
           minion.remote_user_name = minion_user_name
-          # TODO:  sync all machine options here, automatically
-        end
-        # ap minion
-        # puts "*"*175
-        # ap minion.roles
-        # ap item.role
-        minion.roles << item.role
-        # ap minion.roles
+          minion.machine_options = machine
 
-        # if item.role_index?
-        #   minions
-        # end
-        # ap minion
+          minion.all_data = data
+          minion.project = project
+          minion.stage = stage
+          minion.default_cloud_provider = data.default_cloud_provider
+          minion.cloud = ::Pvcglue.cloud
+        end
 
-        minion.all_data = data
-        minion.project = project
-        minion.stage = stage
-        minion.cloud_provider = data.cloud_provider
-        minion.cloud = ::Pvcglue.cloud
+        minion.roles << item.role
 
         minions[machine.name] = minion
       end

data/lib/pvcglue/cloud_providers.rb

@@ -0,0 +1,34 @@
+module Pvcglue
+  class CloudProviders
+    # REQUIRED_OPTIONS = []
+
+    def self.init(provider_options)
+      @options = provider_options
+      @name = provider_options.name
+      if provider_options.name == 'digital-ocean'
+        Pvcglue.logger.debug("Digital Ocean provider initialized for '#{provider_options.name}'.")
+        Pvcglue::CloudProviders::DigitalOcean.new(provider_options)
+      elsif provider_options.name == 'linode'
+        Pvcglue.logger.debug("Linode provider initialized for '#{provider_options.name}'.")
+        Pvcglue::CloudProviders::Linode.new(provider_options)
+      else
+        raise(Thor::Error, "Cloud Provider '#{provider_options.name}' not supported, use 'manual' mode.")
+      end
+    end
+
+    def name
+      @name
+    end
+
+   def options
+      @options
+    end
+
+    def validate_options!(options)
+      errors = []
+      REQUIRED_OPTIONS.each { |option_name| errors << "#{option_name} required" unless options[option_name] }
+      raise("Errors:  #{errors.join(', ')}.") if errors.any?
+    end
+
+  end
+end

data/lib/pvcglue/cloud_providers/digital_ocean.rb

@@ -0,0 +1,79 @@
+module Pvcglue
+  class CloudProviders
+    REQUIRED_OPTIONS = %w(name region capacity image)
+
+    class DigitalOcean < Pvcglue::CloudProviders
+
+      def initialize(provider_options)
+        @options = provider_options
+      end
+
+      def create(options)
+        validate_options!(options)
+
+        opts = options.to_h
+        opts[:size] = opts.delete('capacity')
+
+        droplet_options = DropletKit::Droplet.new(opts)
+        droplet = client.droplets.create(droplet_options)
+        Pvcglue.logger.debug("Created Digital Ocean droplet, ID:  #{droplet.id}")
+        droplet
+      end
+
+      def ready?(minion)
+        droplet = find_by_name(minion.machine_name)
+
+        droplet.raw_data[:status] == 'active'
+      end
+
+      def find_by_name(minion_name)
+        result = droplets.detect { |droplet| droplet.name == minion_name }
+        normalize_machine_data(result) if result
+      end
+
+      def reload_machine_data(minion)
+        find_by_name(minion.machine_name)
+      end
+
+      def normalize_machine_data(droplet)
+        machine = ::SafeMash.new
+        machine.raw_data = droplet
+        machine.id = droplet.id.to_s
+        machine.public_ip = get_public_ip(droplet)
+        machine.private_ip = get_private_ip(droplet)
+        machine.cloud_provider = 'digital-ocean'
+        machine
+      end
+
+      def get_private_ip(droplet)
+        get_ip_addresses(droplet).private
+      end
+
+      def get_public_ip(droplet)
+        get_ip_addresses(droplet).public
+      end
+
+      def droplets
+        client.droplets.all
+        # @droplets ||= client.droplets.all
+      end
+
+      def client
+        @client ||= get_client
+      end
+
+      def get_client
+        access_token = YAML::load(File.open(File.join(ENV['HOME'], '.config/doctl/config.yaml')))['access-token']
+        ::DropletKit::Client.new(access_token: access_token)
+      end
+
+      def get_ip_addresses(droplet)
+        ips = ::SafeMash.new
+        droplet.networks.v4.each do |network|
+          ips[network.type] = network.ip_address
+        end
+        ips
+      end
+    end
+  end
+end

data/lib/pvcglue/cloud_providers/linode.rb

@@ -0,0 +1,129 @@
+module Pvcglue
+  class CloudProviders
+    REQUIRED_OPTIONS = %w(name region capacity image)
+
+    class Linode < Pvcglue::CloudProviders
+
+      def create(options)
+        validate_options!(options)
+
+        cmd = "linode create #{options.name} "
+        cmd += "--location #{options.region} "
+        cmd += "--plan #{options.capacity} "
+        cmd += '--payment-term 1 '
+        cmd += "--distribution '#{options.image}' "
+        cmd += "--group '#{options.group}' " if options.group
+        cmd += "--password #{generate_password} "
+        cmd += "--pubkey-file '#{File.expand_path('~/.ssh/id_rsa.pub')}' "
+        cmd += '--json'
+
+        data, error = run(cmd)
+        raise(error) if error.present?
+
+        cmd = "linode ip-add --label #{options.name} --private "
+        cmd += '--json'
+        data, error = run(cmd)
+        raise(error) if error.present?
+
+        #<SafeMash staging-lb=#<SafeMash job="start" jobid=45919584 message="Completed. Booting staging-lb..." request_action="create" request_error="">>
+        machine = find_by_name(options.name)
+        Pvcglue.logger.debug("Created Linode, ID:  #{machine.id}")
+        machine
+      end
+
+      def generate_password
+        password = "#{SecureRandom.hex}a1" # ensure required 'character classes'
+        Pvcglue.verbose? { password }
+        password
+      end
+
+      def reload_machine_data(minion)
+        find_by_name(minion.machine_name)
+      end
+
+      def ready?(minion)
+        machine = find_by_name(minion.machine_name)
+        machine.raw_data.values.first.status == 'running'
+      end
+
+      def find_by_name(name)
+        # return nil
+        cmd = "linode show --label #{name} "
+        cmd += '--json'
+
+        data, error = run(cmd)
+        if error =~ /Couldn't find/
+          return nil
+        elsif error.present?
+          raise(error)
+        end
+        normalize_machine_data(data)
+      end
+
+=begin
+> linode show staging-lb --json
+{
+   "staging-lb" : {
+      "backupsenabled" : false,
+      "totalram" : "1.00GB",
+      "group" : "",
+      "request_error" : "",
+      "ips" : [
+         "192.168.132.194",
+         "104.237.159.149"
+      ],
+      "location" : "fremont",
+      "totalhd" : "20.00GB",
+      "status" : "running",
+      "label" : "staging-lb",
+      "request_action" : "show",
+      "linodeid" : 2817249
+   }
+}
+=end
+      def normalize_machine_data(data)
+        machine = ::SafeMash.new
+        machine.raw_data = data
+        machine.id = data.values.first.linodeid
+        machine.public_ip = get_public_ip(data.values.first.ips)
+        machine.private_ip = get_private_ip(data.values.first.ips)
+        machine.cloud_provider = 'linode'
+        machine
+      end
+
+      def get_private_ip(ips)
+        ips.each { |ip| return ip if is_private?(ip) }
+        nil
+      end
+
+      def get_public_ip(ips)
+        ips.each { |ip| return ip unless is_private?(ip) }
+        nil
+      end
+
+      def is_private?(ip)
+        ip =~ /^(?:10|127|172\.(?:1[6-9]|2[0-9]|3[01])|192\.168)\..*/m
+      end
+
+
+      def run(cmd)
+        result = Pvcglue.system_get_stdout(cmd)
+        data = ::SafeMash.new(JSON.parse(result))
+        Pvcglue.verbose? { data.inspect }
+        [data, request_error(data)]
+      end
+
+      def request_error(data)
+        data.values.first.request_error
+      end
+
+      # def get_ip_addresses(droplet)
+      #   ips = ::SafeMash.new
+      #   droplet.networks.v4.each do |network|
+      #     ips[network.type] = network.ip_address
+      #   end
+      #   ips
+      # end
+    end
+  end
+end