cult 0.1.3.pre → 0.1.4.pre

data/lib/cult/cli/task_cmd.rb

@@ -6,7 +6,7 @@ module Cult
         optional_project
         name        'task'
         aliases     'tasks'
-        summary     'Task Manipulation'
+        summary     'Task manipulation'
         usage       'task [command]'
         description <<~EOD.format_description
           Tasks are basically shell scripts.  Or anything with a \#! line, or
@@ -31,8 +31,9 @@ module Cult
           neatly line these up for you.
         EOD
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           puts cmd.help
+          exit
         end
       end
 
@@ -42,10 +43,10 @@ module Cult
         aliases     'reserial'
         summary     'Resequences task serial numbers'
 
-        flag     :A,  :all,             'Re-sequence all roles'
-        flag     :G,  :'git-add',       '`git add` each change'
-        required :r,  :role,            'Roles to resequence (multiple)',
-                      multiple: true
+        flag     :A,  :all,       'Re-sequence all roles'
+        flag     :G,  :'git-add', '`git add` each change'
+        required :r,  :role,      'Resequence only /NODE+/ (multiple)',
+                                  multiple: true
 
         description <<~EOD.format_description
           Resequences the serial numbers in each task provided with --roles,
@@ -65,7 +66,7 @@ module Cult
         EOD
 
 
-        run(arguments: 0) do |opts, args, cmd|
+        run(arguments: none) do |opts, args, cmd|
           if opts[:all] && Array(opts[:role]).size != 0
             fail CLIError, "can't supply -A and also a list of roles"
           end
@@ -131,24 +132,23 @@ module Cult
       task.add_command task_sanity
 
 
-      task_create = Cri::Command.define do
-        name        'create'
-        aliases     'new'
+      task_new = Cri::Command.define do
+        name        'new'
         usage       'create [options] DESCRIPTION'
         summary     'create a new task for ROLE with a proper serial'
         description <<~EOD.format_description
         EOD
 
-        required :r, :role, 'role for task.  defaults to "all"'
+        required :r, :role, '/ROLE/ for task.  defaults to "base"'
         flag :e, :edit, 'open generated task file in your $EDITOR'
 
         run do |opts, args, cmd|
           english = args.join " "
-          opts[:roles] ||= 'all'
+          opts[:roles] ||= 'base'
           puts [english, opts[:roles], opts[:edit]].inspect
         end
       end
-      task.add_command task_create
+      task.add_command(task_new)
 
       task
     end

data/lib/cult/commander.rb

@@ -18,6 +18,7 @@ module Cult
       Shellwords.escape(s)
     end
 
+
     def send_tar(io, ssh)
       filename = SecureRandom.hex + ".tar"
       puts "Uploading bundle: #{filename}"
@@ -26,6 +27,7 @@ module Cult
       ssh.exec! "tar -xf #{esc(filename)} && rm #{esc(filename)}"
     end
 
+
     def create_build_tar(role)
       io = StringIO.new
       Bundle.new(io) do |bundle|
@@ -41,11 +43,12 @@ module Cult
       io
     end
 
+
     def exec_remote!(ssh:, role:, task:)
       token = SecureRandom.hex
       task_bin = role.relative_path(task.path)
 
-      puts "Executing: #{task.remote_path}"
+      puts "Executing: #{task.remote_path} on #{node.name}"
       res = ssh.exec! <<~BASH
         cd #{esc(role.remote_path)}; \
         ./#{esc(task_bin)} && \
@@ -63,6 +66,7 @@ module Cult
       end
     end
 
+
     def install!(role)
       connect(user: role.user) do |ssh|
         io = create_build_tar(role)
@@ -77,20 +81,28 @@ module Cult
       end
     end
 
-    def find_sync_tasks
+
+    def find_sync_tasks(pass:, roles: nil)
+      selected_roles = node.build_order
+
+      if roles
+        selected_roles.select! { |r| roles.include?(r) }
+      end
+
       r = []
-      node.build_order.each do |role|
-        role.event_tasks.each do |task|
-          r << task if task.name == 'sync'
+      selected_roles.each do |role|
+        r += role.event_tasks.select do |t|
+          t.event == :sync && t.pass == pass
         end
       end
       r
     end
 
-    def create_sync_tar
+
+    def create_sync_tar(pass:, roles: nil)
       io = StringIO.new
       Bundle.new(io) do |bundle|
-        find_sync_tasks.each do |task|
+        find_sync_tasks(pass: pass, roles: roles).each do |task|
           bundle.add_file(project, task.role, node, task)
         end
       end
@@ -99,29 +111,53 @@ module Cult
       io
     end
 
-    def sync!
+
+    def sync!(pass:, roles: nil)
+      io = create_sync_tar(pass: pass, roles: roles)
+      return if io.eof?
+
       connect do |ssh|
-        io = create_sync_tar
         send_tar(io, ssh)
-        find_sync_tasks.each do |task|
+        find_sync_tasks(pass: pass, roles: roles).each do |task|
           exec_remote!(ssh: ssh, role: task.role, task: task)
         end
       end
     end
 
+
     def bootstrap!
       bootstrap_role = CLI.fetch_item('bootstrap', from: Role)
       install!(bootstrap_role)
     end
 
+    def ping
+      connect do |ssh|
+        ssh.exec! "uptime"
+      end
+    rescue
+      nil
+    end
+
     def connect(user: nil, &block)
-      user ||= node.user
-      puts "Connecting with user=#{user}, key=#{node.ssh_private_key_file}"
-      Net::SSH.start(node.host,
-                     user,
-                     keys_only: true,
-                     keys: [node.ssh_private_key_file]) do |ssh|
-        yield ssh
+      5.times do |attempt|
+        begin
+          user ||= node.user
+          puts "Connecting with user=#{user}, host=#{node.host}, " +
+               "key=#{node.ssh_private_key_file}"
+          Net::SSH.start(node.host,
+                         user,
+                         port: node.ssh_port,
+                         user_known_hosts_file: node.ssh_known_hosts_file,
+                         timeout: 5,
+                         auth_methods: ['publickey'],
+                         keys_only: true,
+                         keys: [node.ssh_private_key_file]) do |ssh|
+            return (yield ssh)
+          end
+        rescue Errno::ECONNREFUSED, Net::SSH::ConnectionTimeout
+          puts "Connection refused.  Retrying"
+          sleep attempt * 3
+        end
       end
     end
   end

data/lib/cult/commander_sync.rb

@@ -0,0 +1,29 @@
+module Cult
+  class CommanderSync
+    attr_reader :project, :nodes
+    def initialize(project:, nodes:)
+      @project, @nodes = project, nodes
+    end
+
+    def sync!(roles: nil, passes: nil)
+      roles ||= Cult.project.roles
+      passes ||= required_passes(roles)
+
+      passes.each do |pass|
+        puts Rainbow("Executing pass #{pass}").yellow
+        Cult.paramap(nodes) do |node|
+          c = Commander.new(project: project, node: node)
+          c.sync!(pass: pass, roles: roles)
+        end
+      end
+    end
+
+    def required_passes(roles)
+      # searches through every node and extracts which passes have to be ran
+      # to satisfy every event task
+      nodes.map(&:build_order).flatten.uniq
+           .select { |r| roles.nil? ? true : roles.include?(r) }
+           .map(&:event_tasks).flatten.map(&:pass).uniq.sort
+    end
+  end
+end

data/lib/cult/definition.rb

@@ -1,83 +1,55 @@
-require 'yaml'
 require 'json'
-require 'forwardable'
 
 module Cult
   class Definition
     attr_reader :object
     attr_reader :bag
 
-    extend Forwardable
-    def_delegators :object, :definition_parameters,
-                            :definition_path,
-                            :definition_parents
-
     def initialize(object)
       @object = object
     end
 
+    def definition_parameters
+      object.definition_parameters
+    end
+
+    def definition_path
+      object.definition_path
+    end
+
+    def definition_parents
+      object.definition_parents
+    end
+
     def inspect
       "\#<#{self.class.name} " +
         "object: #{object.inspect}, " +
         "params: #{definition_parameters}, " +
         "parents: #{definition_parents}, " +
-        "direct_values: #{bag}>"
+        "bag: #{bag}>"
     end
     alias_method :to_s, :inspect
 
-
-    def filenames
-      Array(definition_path).map do |dp|
-        attempt = [ "#{dp}.yaml", "#{dp}.yml", "#{dp}.json" ]
-        existing = attempt.select do |filename|
-          File.exist?(filename)
-        end
-        if existing.size > 1
-          raise RuntimeError, "conflicting definition files: #{existing}"
-        end
-        existing[0]
-      end.compact
-    end
-
-
-    def decoder_for(filename)
-      @decoder_for ||= begin
-        case filename
-          when nil
-            nil
-          when /\.json\z/
-            JSON.method(:parse)
-          when /\.ya?ml\z/
-            YAML.method(:safe_load)
-          else
-            fail RuntimeError, "No decoder for file type: #{filename}"
-        end
-      end
-    end
-
-
     def bag
-      @bag ||= begin
-        result = {}
-        filenames.each do |filename|
-          erb = ::Cult::Template.new(project: nil, **definition_parameters)
-          contents = erb.process(File.read(filename), filename: filename)
-          result.merge! decoder_for(filename).call(contents)
-        end
-        result
+      @bag ||= Array(definition_path).select do |filename|
+        File.exist?(filename)
+      end.inject({}) do |acc, filename|
+        erb = ::Cult::Template.new(project: nil, **definition_parameters)
+        contents = erb.process(File.read(filename), filename: filename)
+        JSON.parse(contents).merge(acc)
       end
     end
     alias_method :to_h, :bag
 
 
     def direct(k)
-      fail "Use string keys" unless k.is_a?(String)
+      fail ArgumentError unless k.is_a?(String)
       bag[k]
     end
 
 
     def [](k)
-      fail "Use string keys" unless k.is_a?(String)
+      fail ArgumentError unless k.is_a?(String)
       if bag.key?(k)
         bag[k]
       else

data/lib/cult/driver.rb

@@ -80,7 +80,7 @@ module Cult
     end
 
 
-    def self.new(api_key:)
+    def self.new(*args)
       try_requires!
       super
     end

data/lib/cult/drivers/common.rb

@@ -90,7 +90,8 @@ module Cult
 
         # We don't particularly need the debian codename
         s = s.gsub(/(\d)[\s-]+(\S+)/, '\1') if s.match(/^debian/i)
-        s
+        s = s.gsub(/[\s.]+/, '-')
+        s.downcase
       end
 
 
@@ -100,14 +101,12 @@ module Cult
         times = 0
         total_wait = 0.0
 
-        catch :done do
-          loop do
-            yield times, total_wait
-            sleep wait
-            times += 1
-            total_wait += wait
-            wait *= scale
-          end
+        loop do
+          yield times, total_wait
+          sleep wait
+          times += 1
+          total_wait += wait
+          wait *= scale
         end
       end
 
@@ -115,11 +114,13 @@ module Cult
       # Waits until SSH is available at host.  "available" jsut means
       # "listening"/acceping connections.
       def await_ssh(host)
+        puts "Awaiting sshd on #{host}"
         backoff_loop do
           begin
             sock = connect_timeout(host, 22, 1)
-            throw :done
-          rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED
+            break
+          rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED, Errno::EHOSTUNREACH,
+                 Errno::EHOSTDOWN
             # Nothing, these are expected
           ensure
             sock.close if sock

data/lib/cult/drivers/digital_ocean_driver.rb

@@ -58,7 +58,7 @@ module Cult
         d = nil
         backoff_loop do
           d = client.droplets.find(id: droplet.id)
-          throw :done if d.status == 'active'
+          break if d.status == 'active'
         end
         return d
       end
@@ -77,7 +77,7 @@ module Cult
         transaction do |xac|
           ssh_key_id = upload_ssh_key(file: ssh_public_key)
           xac.rollback do
-            destroy_ssh_key!(id: ssh_key_id)
+            destroy_ssh_key!(ssh_key_id: ssh_key_id)
           end
 
           begin

data/lib/cult/drivers/virtual_box_driver.rb

@@ -0,0 +1,156 @@
+require 'shellwords'
+
+module Cult
+  module Drivers
+    class VirtualBoxDriver < ::Cult::Driver
+
+      def initialize(api_key:)
+      end
+
+
+      def sizes_map
+        [1, 2, 4, 6, 8, 10, 12, 16].map do |size|
+          ["#{size}gb", size * 1024 * 1024]
+        end.to_h
+      end
+      with_id_mapping :sizes_map
+
+
+      def images_map
+        %x(VBoxManage list vms).each_line.map do |line|
+          words = Shellwords.split(line.chomp)
+          [ distro_name(words[0]), words[1] ]
+        end.to_h
+      end
+      with_id_mapping :images_map
+
+
+      def zones
+        ['local']
+      end
+
+
+      def ip_property_name(index, protocol)
+        protocols = {
+          ipv4: 'V4',
+          ipv6: 'V6'
+        }
+        "/VirtualBox/GuestInfo/Net/#{index}/#{protocols[protocol]}/IP"
+      end
+
+
+      def esc(s)
+        Shellwords.escape(s)
+      end
+
+
+      def unset_ip_data(name, index, protocol)
+        cmd = "VBoxManage guestproperty unset #{esc(name)} " +
+              "#{ip_property_name(index, protocol)}"
+        `#{cmd}`
+      end
+
+
+      def get_ip_data(name, index, protocol)
+        cmd = "VBoxManage guestproperty get #{esc(name)} " +
+              "#{ip_property_name(index, protocol)}"
+        s = `#{cmd}`
+
+        if $?.success? && (m = s.match(/^Value: (.+)$/))
+          m[1]
+        else
+          nil
+        end
+      end
+
+
+      def await_ip_address(name, index, protocol)
+        puts "Awaiting IP address from VirtualBox Guest Additions"
+        unset_ip_data(name, index, protocol)
+
+        backoff_loop do
+          if (ip = get_ip_data(name, index, protocol))
+            return ip
+          end
+        end
+      end
+
+
+      def destroy!(id:, ssh_key_id:)
+        system 'VBoxManage', 'controlvm', id, 'poweroff'
+        system 'VBoxManage', 'unregistervm', id, '--delete'
+      end
+
+
+      def guest_copy(name, src, dst)
+        cmd = "VBoxManage guestcontrol #{esc(name)} " +
+              "--username root --password password " +
+              "copyto #{esc(src)} --target-directory #{esc(dst)}"
+        puts cmd
+        `#{cmd}`
+      end
+
+
+      def guest_command(name, cmd)
+        cmd = "VBoxManage guestcontrol #{esc(name)} " +
+              "--username root --password password " +
+              "run -- /bin/sh -c #{esc(cmd)}"
+        puts cmd
+        `#{cmd}`
+      end
+
+
+      def provision!(name:, size:, zone:, image:, ssh_public_key:)
+        system 'VBoxManage', 'clonevm',
+                fetch_mapped(name: :image, from: images_map, key: image),
+               '--name', name, '--register'
+
+        system 'VBoxManage', 'modifyvm', name, '--groups', '/Cult'
+        system 'VBoxManage', 'startvm', name, '--type', 'headless'
+
+        public_ip  = await_ip_address(name, 0, :ipv4)
+        private_ip = public_ip
+
+        await_ssh(public_ip)
+
+        guest_command(name, "mkdir -m 0600 /root/.ssh")
+        guest_copy(name, ssh_public_key, "/root/.ssh/authorized_keys")
+        guest_command(name, "chmod 0644 /root/.ssh/authorized_keys")
+        guest_command(name, "passwd -l root")
+
+        return {
+            name:          name,
+            size:          size,
+            zone:          zone,
+            image:         image,
+
+            id:           name,
+            created_at:   Time.now.iso8601,
+            host:         public_ip,
+            ipv4_public:  public_ip,
+            ipv4_private: private_ip,
+            ipv6_public:  nil,
+            ipv6_private: nil,
+            meta:         {}
+        }
+      end
+
+      def self.setup!
+        super
+
+        inst = new(api_key: nil)
+
+        return {
+          driver: driver_name,
+          api_key: nil,
+          configurations: {
+            sizes:  inst.sizes,
+            zones:  inst.zones,
+            images: inst.images,
+          }
+        }
+      end
+
+    end
+  end
+end

data/lib/cult/drivers/vultr_driver.rb

@@ -31,7 +31,7 @@ module Cult
 
 
       def zones_map
-        Vultr::Region.list[:result].map do |k, v|
+        Vultr::Regions.list[:result].map do |k, v|
           [slugify(v["regioncode"]), v["DCID"]]
         end.to_h
       end
@@ -57,7 +57,7 @@ module Cult
 
 
       def sizes_map
-        Vultr::Plan.list[:result].values.select do |v|
+        Vultr::Plans.list[:result].values.select do |v|
           v["plan_type"] == 'SSD'
         end.map do |v|
           if (m = v["name"].match(/^(\d+) ([MGTP]B) RAM/i))
@@ -141,7 +141,7 @@ module Cult
           # Wait until it's active, it won't have an IP until then
           backoff_loop do
             r = Vultr::Server.list(SUBID: subid)[:result]
-            throw :done if r['status'] == 'active'
+            break if r['status'] == 'active'
           end
 
           iplist4 = Vultr::Server.list_ipv4(SUBID: subid)[:result].values[0]