pvcglue 0.9.4 → 0.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8640111473b3ed136163c0bb0fb2a7af3aca750
-  data.tar.gz: c509475f64ca8e936b661d238840be07b1ba435c
+  metadata.gz: caa789ef881164fe12fd0afa7ea98479ab5a70e4
+  data.tar.gz: 1cb97f3b5a5d13984e2b135652a722e149bf63bb
 SHA512:
-  metadata.gz: 6880a84309e9d85cb6cd8e4965620f35672e91e55a7ce614e79b97dbba5041070ce6b53bc19f7f9e6a7842d4af0e6bd7d76e119a07e8b90906bdc0e2be3c8955
-  data.tar.gz: 47f8cded36079c27e0db96acca25db1ebfaecb273a95bff1ee42c0e8ba5c61e79b7bef22cee4685c94dd538008059d7604d45ffda41298c6a20c4ec36d5eb404
+  metadata.gz: f22dc9c678ca9edf6ffb7b88183e7e047334f60e9d83f8084c08f55350b70e3424350f05de2d86c0193c3b17eeb3a816b5fc0a88db3df46459a8290fe6403ff9
+  data.tar.gz: 32480c58b7d164a511b6c5727d23d22f2a6b0c7059d0f6a125baf9f59acfcce278a67922a654fc8d9c253291f6363839a14eed13b5c95f709f6e2cd7eb50a2b3

data/README.md

@@ -4,7 +4,9 @@ The "glue" that creates a tightly integrated (and very small) virtual cloud for
 
 PVC Glue is an cloud application manager for Rails applications using your own (virtual) servers.
 
-PVC Glue was developed as a professional grade replacement for Heroku (and others).
+PVC Glue was developed as a professional grade replacement for Heroku (and others).  It is 
+designed to be used on small to medium size applications, depending on the application 
+requirements and hardware used.
 
 ![pvcglue diagram](/../master/images/pvcglue.png?raw=true "PVC Glue Server Diagram")
 
@@ -12,7 +14,7 @@ Currently supported stack:
 
   * SSL support:  none, manual and automatic with Let's Encrypt
   * Ubuntu 16.04 LTS
-  * Provision servers automatically on Digital Ocean (and Linode*)
+  * Provision servers automatically on Digital Ocean and Linode
   * No need to install anything on servers first (you just need SSH access)
   * Ruby >= 1.9 (multiple versions supported on same server!)
   * Rails >= 3.2
@@ -21,11 +23,12 @@ Currently supported stack:
   * Nginx
   * Phusion Passenger (>= 5.x)
   * Memcached*
-  * Redis*
-  * Designed to easily support multiple staging environments
+  * Redis
+  * Designed to easily manage multiple environments (i.e. alpha, beta, preview, production ...)
 
 Workers:
 
+  * Sidekiq
   * Delayed Job*
   * Rescue*
   

data/bin/pvc

@@ -4,7 +4,6 @@ require 'benchmark'
 
 # Allow use of Capistrano style environment syntax and convert to 'standard' syntax
 # Example:  `pvc production bootstrap` ==> `pvc bootstrap --stage=production`
-# TODO: refactor to use a list of user specified environments
 
 Pvcglue.logger.info('Starting up...')
 # Pvcglue.logger.info("----- Done #{Benchmark.measure { Pvcglue::CLI.start }}")
@@ -12,7 +11,8 @@ Pvcglue.logger.info('Starting up...')
 def capistrano_style_environment
   Pvcglue.logger.debug { ARGV.inspect }
   if ARGV.count >= 2
-    if %w[local vmtest test alpha beta gamma delta preview production staging].include?(ARGV[0])
+    # TODO: refactor to use a list of user specified environments
+    if %w[local vmtest test alpha beta gamma delta preview production staging runner].include?(ARGV[0])
       ARGV[0], ARGV[1] = ARGV[1], "--stage=#{ARGV[0]}"
       Pvcglue.logger.debug { ARGV.inspect }
       Pvcglue::CLI.start

data/lib/pvcglue.rb

@@ -25,7 +25,7 @@ require 'hashie'
 require 'pvcglue/custom_hashie'
 require 'pvcglue/minion'
 require 'pvcglue/cloud_providers'
-require 'droplet_kit'
+# require 'droplet_kit'
 # require 'pvcglue/digital_ocean'
 require 'logger'
 require 'pvcglue/connection'
@@ -215,11 +215,12 @@ module Pvcglue
     true
   end
 
-  def self.system_get_stdout(cmd)
+  def self.system_get_stdout(cmd, raise_error = false)
     Pvcglue.logger.debug { cmd }
     result = `#{cmd}`
     Pvcglue.verbose? { result }
     Pvcglue.logger.debug { "exit_code=#{$?.to_i}" }
+    raise($?.inspect) if raise_error && $?.to_i != 0
     result
   end
 

data/lib/pvcglue/cli.rb

@@ -61,11 +61,46 @@ module Pvcglue
       Pvcglue::Stack.build(Pvcglue.cloud.minions, roles)
     end
 
+    desc "show configuration", "Show configuration details for the stage"
+    method_option :stage, :required => true, :aliases => "-s"
+
+    def config(roles = 'all')
+      if roles == 'all'
+        ap Pvcglue.cloud.data
+      else
+        minions = Pvcglue.cloud.minions_filtered(roles)
+        minions.each do |minion_name, minion|
+          filtered = '[filtered]'
+          # filtered = nil
+          project = minion.merge({
+            stages: filtered,
+            all_data: filtered,
+            cloud: filtered,
+            project: filtered,
+            stage: filtered,
+            default_cloud_provider: filtered,
+            connection: filtered
+          })
+          # project = project.select { |_, value| !value.nil? }
+          data = minion.merge({
+            cloud: filtered,
+            all_data: filtered,
+            connection: filtered,
+            project: project
+          })
+          # ap data.select { |_, value| !value.nil? }
+          ap(data, indent: 2)
+        end
+
+      end
+
+    end
+
     desc "console", "open rails console"
     method_option :stage, :required => true, :aliases => "-s"
 
-    def console(server='web')
-      data = Pvcglue.cloud.minions_filtered(server)
+    def console(role='web')
+      data = Pvcglue.cloud.minions_filtered(role)
       minion_name = data.keys.first
       minion = data.values.first
       working_dir = Pvcglue.cloud.deploy_to_app_current_dir

data/lib/pvcglue/cloud.rb

@@ -365,6 +365,10 @@ module Pvcglue
       end
     end
 
+    def generated_file_warning
+      'This is a generated file.  Do not modify...or else!  :)'
+    end
+
     def db_rebuild
       !!stage[:db_rebuild]
     end
@@ -446,6 +450,13 @@ module Pvcglue
       '/var/www/letsencrypt_root'
     end
 
+    def service_directory
+      '/lib/systemd/system/'
+    end
+
+    def service_extension
+      '.service'
+    end
 
     # ==============================================================================================
 
@@ -495,7 +506,8 @@ module Pvcglue
 
 
     def minion_user_name_base
-      project[:user_name_base] || 'deploy'
+      # project[:user_name_base] || 'deploy'
+      project[:user_name_base] || raise('user_name_base is required')
     end
 
     def minion_user_name
@@ -543,6 +555,7 @@ module Pvcglue
       find_or_raise(data.machines, name)
     end
 
+
     def get_minions
       minions = ::SafeMash.new
       stage.stack.each do |item|
@@ -564,6 +577,7 @@ module Pvcglue
           minion.cloud_id = machine.cloud_id
           minion.remote_user_name = minion_user_name
           minion.machine_options = machine
+          minion.stage_options = item
 
           minion.all_data = data
           minion.project = project

data/lib/pvcglue/cloud_providers.rb

@@ -1,8 +1,10 @@
+# TODO:  Refactor this, it's kinda messy :(
 module Pvcglue
   class CloudProviders
     # REQUIRED_OPTIONS = []
 
     def self.init(provider_options)
+
       @options = provider_options
       @name = provider_options.name
       if provider_options.name == 'digital-ocean'
@@ -24,9 +26,9 @@ module Pvcglue
       @options
     end
 
-    def validate_options!(options)
+    def validate_options!(options, required)
       errors = []
-      REQUIRED_OPTIONS.each { |option_name| errors << "#{option_name} required" unless options[option_name] }
+      required.each { |option_name| errors << "#{option_name} required" unless options[option_name] }
       raise("Errors:  #{errors.join(', ')}.") if errors.any?
     end
 

data/lib/pvcglue/cloud_providers/digital_ocean.rb

@@ -1,7 +1,5 @@
 module Pvcglue
   class CloudProviders
-    REQUIRED_OPTIONS = %w(name region capacity image)
-
     class DigitalOcean < Pvcglue::CloudProviders
 
       def initialize(provider_options)
@@ -9,17 +7,44 @@ module Pvcglue
       end
 
       def create(options)
-        validate_options!(options)
-
-        opts = options.to_h
-        opts[:size] = opts.delete('capacity')
-
-        droplet_options = DropletKit::Droplet.new(opts)
-        droplet = client.droplets.create(droplet_options)
+        validate_options!(options, %w(name region capacity image))
+        #doctl compute droplet create test --size 512mb --image ubuntu-16-04-x64 --region sfo2 --ssh-keys d1:fe:e8:53:d4:fb:eb:f1:db:fc:ef:18:f1:cf:1e:5d --enable-backups --enable-private-networking
+        cmd = "doctl compute droplet create #{options.name} "
+        cmd += "--region #{options.region} "
+        cmd += "--size #{options.capacity} "
+        cmd += "--image '#{options.image}' "
+        cmd += '--enable-private-networking '
+        cmd += '--enable-monitoring '
+        cmd += '--enable-backups ' if options.backups
+        # cmd += "--tags '#{options.group}' " if options.group
+        cmd += "--ssh-keys #{options.ssh_keys.join(',')} "
+        cmd += '--output json'
+
+        result = Pvcglue.system_get_stdout(cmd, true)
+        array_data = JSON.parse(result)
+        array_data.each { |h| h['size_data'] = h.delete('size') }
+        data = []
+        array_data.each do |machine|
+          data << ::SafeMash.new(machine)
+        end
+        droplet = data.first
+        Pvcglue.verbose? { data.inspect }
         Pvcglue.logger.debug("Created Digital Ocean droplet, ID:  #{droplet.id}")
         droplet
       end
 
+      # def create(options)
+      #   validate_options!(options, %w(name region capacity image))
+      #   byebug
+      #   opts = options.to_h
+      #   opts[:size] = opts.delete('capacity')
+      #
+      #   droplet_options = DropletKit::Droplet.new(opts)
+      #   droplet = client.droplets.create(droplet_options)
+      #   Pvcglue.logger.debug("Created Digital Ocean droplet, ID:  #{droplet.id}")
+      #   droplet
+      # end
+      #
       def ready?(minion)
         droplet = find_by_name(minion.machine_name)
 
@@ -53,9 +78,40 @@ module Pvcglue
         get_ip_addresses(droplet).public
       end
 
+      def run(cmd)
+        result = Pvcglue.system_get_stdout(cmd, true)
+        array_data = JSON.parse(result)
+        array_data.each { |h| h['size_data'] = h.delete('size') }
+        data = []
+        array_data.each do |machine|
+          data << ::SafeMash.new(machine)
+        end
+        Pvcglue.verbose? { data.inspect }
+        [data, request_error(data)]
+      end
+
+      def request_error(data)
+        # TODO:  Better error handling
+        return nil # for now
+        # return nil unless data.errors
+        # raise data.errors.values.join('.  ')
+      end
+
       def droplets
-        client.droplets.all
+        # client.droplets.all
         # @droplets ||= client.droplets.all
+        # return nil
+        cmd = 'doctl compute droplet list --output json'
+
+        result = Pvcglue.system_get_stdout(cmd, true)
+        array_data = JSON.parse(result)
+        array_data.each { |h| h['size_data'] = h.delete('size') }
+        data = []
+        array_data.each do |machine|
+          data << ::SafeMash.new(machine)
+        end
+        Pvcglue.verbose? { data.inspect }
+        data
       end
 
       def client

data/lib/pvcglue/cloud_providers/linode.rb

@@ -1,11 +1,14 @@
 module Pvcglue
   class CloudProviders
-    REQUIRED_OPTIONS = %w(name region capacity image)
-
     class Linode < Pvcglue::CloudProviders
 
+      def initialize(provider_options)
+        @options = provider_options
+      end
+
       def create(options)
-        validate_options!(options)
+        # TODO:  Set machine name.  Example:  "thekraken-alpha@ubuntu:~$" should be "thekraken-alpha@staging-web:~$"
+        validate_options!(options, %w(name region capacity image))
 
         cmd = "linode create #{options.name} "
         cmd += "--location #{options.region} "

data/lib/pvcglue/connection.rb

@@ -162,10 +162,10 @@ module Pvcglue
 
     end
 
-    def write_to_file_from_template(user, template_file_name, file, owner = nil, group = nil, permissions = nil)
+    def write_to_file_from_template(user, template_file_name, file, locals = nil, owner = nil, group = nil, permissions = nil)
       Pvcglue.logger.debug { "Writing to #{file} from template '#{template_file_name}'" }
       template = Tilt.new(Pvcglue.template_file_name(template_file_name))
-      data = template.render(self, minion: minion)
+      data = template.render(self, {minion: minion}.merge(locals || {}))
 
       write_to_file(user, data, file, owner, group, permissions)
     end

data/lib/pvcglue/custom_hashie.rb

@@ -1,3 +1,8 @@
 class SafeMash < Hashie::Mash
   include Hashie::Extensions::Mash::SafeAssignment
 end
+
+# class OverrideMash < Hashie::Mash
+#   disable_warnings
+#   include Hashie::Extensions::MethodAccessWithOverride
+# end

data/lib/pvcglue/docs.rb

@@ -57,7 +57,10 @@ module Pvcglue
     end
 
     def set_item(options)
-      return unless enabled
+      unless enabled
+        yield
+        return
+      end
       options = ::SafeMash.new(options)
       add_header(3, options.heading)
       add('----')

data/lib/pvcglue/env.rb

@@ -93,38 +93,48 @@ module Pvcglue
       defaults['RAILS_SECRET_TOKEN'] = SecureRandom.hex(64) # From rails/railties/lib/rails/tasks/misc.rake
       defaults['SECRET_KEY_BASE'] = SecureRandom.hex(64) # From rails/railties/lib/rails/tasks/misc.rake
 
-      if Pvcglue.cloud.minions_filtered('pg').any?
+      # Don't set a default until we have a provisioned server
+      if db_host
         defaults['DB_USER_POSTGRES_HOST'] = db_host
-        defaults['DB_USER_POSTGRES_PORT'] = "5432"
+        defaults['DB_USER_POSTGRES_PORT'] = '5432'
         defaults['DB_USER_POSTGRES_USERNAME'] = "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}"
         defaults['DB_USER_POSTGRES_DATABASE'] = "#{Pvcglue.cloud.app_name}_#{Pvcglue.cloud.stage_name_validated}"
         defaults['DB_USER_POSTGRES_PASSWORD'] = new_password
       end
 
-      if Pvcglue.cloud.minions_filtered('memcached').any?
+      # Don't set a default until we have a provisioned server
+      if memcached_host
         defaults['MEMCACHE_SERVERS'] = memcached_host
       end
 
-      if Pvcglue.cloud.minions_filtered('redis').any?
-        defaults['REDIS_SERVER'] = redis_host
+      # Don't set a default until we have a provisioned server
+      if redis_url
+        defaults['REDIS_URL'] = redis_url
       end
 
       defaults
     end
 
     def self.db_host
-      # Assume 1 pg server
-      Pvcglue.cloud.minions_filtered('pg').values.first.private_ip
+      # Assume first pg server
+      minion = Pvcglue.cloud.minions_filtered('pg').values.first
+      minion && minion.private_ip
     end
 
     def self.memcached_host
-      node = Pvcglue.cloud.find_minion_by_name('memcached', false)
-      node ? "#{node['memcached']['private_ip']}:11211" : ""
-    end
-
-    def self.redis_host
-      node = Pvcglue.cloud.find_minion_by_name('redis', false)
-      node ? "#{node['redis']['private_ip']}:6379" : ""
+      # Assume first server
+      minion = Pvcglue.cloud.minions_filtered('memcached').values.first
+      ip = minion && minion.private_ip
+      ip ? "#{ip}:11211" : nil
+    end
+
+    def self.redis_url
+      # Assume first server
+      # TODO:  (low priority) check all other apps/environments to see if
+      # the current database number is used, and find an unused one, if it is.
+      minion = Pvcglue.cloud.minions_filtered('redis').values.first
+      ip = minion && minion.private_ip
+      ip ? "redis://#{ip}:6379/0" : nil
     end
 
     def self.new_password

data/lib/pvcglue/manager.rb

@@ -4,6 +4,7 @@ module Pvcglue
 
   class Manager < Thor
 
+    # TODO:  Rename to 'build'?
     desc "bootstrap", "bootstrap"
 
     def bootstrap
@@ -48,12 +49,9 @@ module Pvcglue
     desc "shell", "run shell"
 
     def sh # `shell` is a Thor reserved word
-      working_dir = self.class.manager_dir
-      cloud_manager = Pvcglue.configuration.cloud_manager
-      user_name = self.class.user_name
-      cloud_name = Pvcglue.configuration.cloud_name
-      puts "Connection to #{cloud_name} cloud on manager at (#{cloud_manager}) as user '#{user_name}'..."
-      system(%(ssh -p #{Pvcglue.cloud.port_in_context(:manager)} -t #{user_name}@#{cloud_manager} "cd #{working_dir} && bash -i"))
+      minion = Pvcglue.cloud.manager_minion
+      Pvcglue.logger.warn("Connecting to #{minion.machine_name} (#{minion.public_ip}) as user '#{minion.remote_user_name}'...")
+      minion.connection.ssh!(minion.remote_user_name, '', '')
     end
 
     desc "user PATH_TO_FILE", "add or update user's ssh key to allow access to the manager"