cult 0.1.1.pre → 0.1.2.pre

data/lib/cult/commander.rb

@@ -2,33 +2,9 @@ require 'net/ssh'
 require 'net/scp'
 require 'shellwords'
 require 'rainbow'
-require 'rubygems/package'
-require 'rubygems/package/tar_writer'
+require 'securerandom'
 
 module Cult
-
-  class Bundle
-    attr_reader :tar
-    def initialize(io, &block)
-      @tar = Gem::Package::TarWriter.new(io)
-      if block_given?
-        begin
-          yield self
-        ensure
-          @tar.close
-          @tar = nil
-        end
-      end
-    end
-
-    def add_file(project, role, node, transferable)
-      data = transferable.contents(project, role, node, pwd: role.path)
-      tar.add_file(transferable.remote_path, transferable.file_mode) do |io|
-        io.write(data)
-      end
-    end
-  end
-
   class Commander
     attr_reader :project
     attr_reader :node
@@ -42,59 +18,109 @@ module Cult
       Shellwords.escape(s)
     end
 
-    def send_bundle(ssh, role)
+    def send_tar(io, ssh)
+      filename = SecureRandom.hex + ".tar"
+      puts "Uploading bundle: #{filename}"
+      scp = Net::SCP.new(ssh)
+      scp.upload!(io, filename)
+      ssh.exec! "tar -xf #{esc(filename)} && rm #{esc(filename)}"
+    end
+
+    def create_build_tar(role)
       io = StringIO.new
       Bundle.new(io) do |bundle|
         puts "Building bundle..."
         role.build_order.each do |r|
-          (r.artifacts + r.tasks).each do |transferable|
+          (r.artifacts + r.build_tasks).each do |transferable|
             bundle.add_file(project, r, node, transferable)
           end
         end
       end
-      filename = "cult-#{role.name}.tar"
-      puts "Uploading bundle #{filename}..."
 
-      scp = Net::SCP.new(ssh)
       io.rewind
-      scp.upload!(io, filename)
-      ssh.exec! "tar -xf #{esc(filename)} && rm #{esc(filename)}"
+      io
+    end
+
+    def exec_remote!(ssh:, role:, task:)
+      token = SecureRandom.hex
+      task_bin = role.relative_path(task.path)
+
+      puts "Executing: #{task.remote_path}"
+      res = ssh.exec! <<~BASH
+        cd #{esc(role.remote_path)}; \
+        ./#{esc(task_bin)} && \
+        echo #{esc(token)}
+      BASH
+
+      if res.chomp.end_with?(token)
+        res = res.gsub(token, '')
+        puts Rainbow(res.gsub(/^/, '    ')).darkgray.italic
+        true
+      else
+        puts Rainbow(res).red
+        puts "Failed"
+        false
+      end
     end
 
     def install!(role)
-      connect(user: role.definition['user']) do |ssh|
-        send_bundle(ssh, role)
+      connect(user: role.user) do |ssh|
+        io = create_build_tar(role)
+        send_tar(io, ssh)
 
         role.build_order.each do |r|
           puts "Installing role: #{Rainbow(r.name).blue}"
-          working_dir = r.remote_path
-          r.tasks.each do |t|
-            puts "Executing: #{t.remote_path}"
-            task_bin = r.relative_path(t.path)
-            res = ssh.exec! <<~BASH
-              cd #{esc(working_dir)}; \
-                if [ ! -f ./#{esc(task_bin)}.success ]; then  \
-                  touch ./#{esc(task_bin)}.attempt && \
-                  ./#{esc(task_bin)} && \
-                  mv ./#{esc(task_bin)}.attempt ./#{esc(task_bin)}.success; \
-                fi
-            BASH
-            unless res.empty?
-              puts Rainbow(res.gsub(/^/, '    ')).darkgray.italic
-            end
+          r.build_tasks.each do |task|
+            exec_remote!(ssh: ssh, role: r, task: task)
           end
         end
       end
     end
 
+    def find_sync_tasks
+      r = []
+      node.build_order.each do |role|
+        role.event_tasks.each do |task|
+          r << task if task.name == 'sync'
+        end
+      end
+      r
+    end
+
+    def create_sync_tar
+      io = StringIO.new
+      Bundle.new(io) do |bundle|
+        find_sync_tasks.each do |task|
+          bundle.add_file(project, task.role, node, task)
+        end
+      end
+
+      io.rewind
+      io
+    end
+
+    def sync!
+      connect do |ssh|
+        io = create_sync_tar
+        send_tar(io, ssh)
+        find_sync_tasks.each do |task|
+          exec_remote!(ssh: ssh, role: task.role, task: task)
+        end
+      end
+    end
+
     def bootstrap!
       bootstrap_role = CLI.fetch_item('bootstrap', from: Role)
       install!(bootstrap_role)
     end
 
-    def connect(user:, &block)
-      puts "Connecting with user=#{user}"
-      Net::SSH.start(node.host, user) do |ssh|
+    def connect(user: nil, &block)
+      user ||= node.user
+      puts "Connecting with user=#{user}, key=#{node.ssh_private_key_file}"
+      Net::SSH.start(node.host,
+                     user,
+                     keys_only: true,
+                     keys: [node.ssh_private_key_file]) do |ssh|
         yield ssh
       end
     end

data/lib/cult/definition.rb

@@ -2,23 +2,20 @@ require 'yaml'
 require 'json'
 require 'forwardable'
 
-require 'cult/template'
-
 module Cult
   class Definition
     attr_reader :object
     attr_reader :bag
 
     extend Forwardable
-    def_delegators :object, :definition_parameters, :definition_path,
+    def_delegators :object, :definition_parameters,
+                            :definition_path,
                             :definition_parents
 
-
     def initialize(object)
       @object = object
     end
 
-
     def inspect
       "\#<#{self.class.name} " +
         "object: #{object.inspect}, " +
@@ -63,8 +60,8 @@ module Cult
       @bag ||= begin
         result = {}
         filenames.each do |filename|
-          erb = Template.new(definition_parameters)
-          contents = erb.process(File.read(filename))
+          erb = ::Cult::Template.new(project: nil, **definition_parameters)
+          contents = erb.process(File.read(filename), filename: filename)
           result.merge! decoder_for(filename).call(contents)
         end
         result

data/lib/cult/driver.rb

@@ -1,8 +1,8 @@
-require 'cult/definition'
 require 'cult/drivers/common'
 
 module Cult
   class Driver
+    include ::Cult::Drivers::Common
 
     # This is raised when a Driver is instantiated,  but the required
     # gems are not installed.

data/lib/cult/drivers/common.rb

@@ -1,10 +1,12 @@
 require 'socket'
 require 'net/ssh'
 
+require 'cult/transaction'
+
 module Cult
   module Drivers
-
     module Common
+
       module ClassMethods
         # Lets us write a method "something_map" that returns {'ident' => ...},
         # and also get a function "something" that returns the keys.
@@ -37,11 +39,6 @@ module Cult
       end
 
 
-      def self.included(cls)
-        cls.extend(ClassMethods)
-      end
-
-
       # works with with_id_mapping to convert a human-readible/normalized key
       # to the id the backend service expects.  Allows '=value' to force a
       # literal value, and gives better error messages.
@@ -81,21 +78,6 @@ module Cult
       end
 
 
-      # Enter this block once a node has been created.  It makes sure it's
-      # destroyed if there's an error later in the procedure.
-      def rollback_on_error(id:, &block)
-        begin
-          yield
-        rescue Exception => e
-          begin
-            destroy!(id: id)
-          ensure
-            raise e
-          end
-        end
-      end
-
-
       def slugify(s)
         s.gsub(/[^a-z0-9]+/i, '-').gsub(/(^\-)|(-\z)/, '').downcase
       end
@@ -187,6 +169,11 @@ module Cult
         end
       end
 
+      def self.included(cls)
+        cls.extend(ClassMethods)
+        cls.include(::Cult::Transaction)
+      end
+
     end
   end
 end

data/lib/cult/drivers/digital_ocean_driver.rb

@@ -1,6 +1,3 @@
-require 'cult/driver'
-require 'cult/drivers/common'
-require 'net/ssh'
 require 'json'
 
 module Cult
@@ -9,8 +6,6 @@ module Cult
     class DigitalOceanDriver < ::Cult::Driver
       self.required_gems = 'droplet_kit'
 
-      include Common
-
       attr_reader :client
 
       def initialize(api_key:)
@@ -49,24 +44,13 @@ module Cult
       with_id_mapping :zones_map
 
 
-      def ssh_keys
-        client.ssh_keys.all.to_a.map(&:to_h)
-      end
-      memoize :ssh_keys
-
 
       def upload_ssh_key(file:)
         key = ssh_key_info(file: file)
         # If we already have one with this fingerprint, use it.
-        if (exist = ssh_keys.find {|dk| dk[:fingerprint] == key[:fingerprint]})
-          exist
-        else
-          ssh_keys_dememo!
-          client.ssh_keys.create \
-            DropletKit::SSHKey.new(fingerprint: key[:fingerprint],
-                                   public_key: key[:data],
-                                   name: key[:name])
-        end
+        dk_key = DropletKit::SSHKey.new(public_key: key[:data],
+                                        name: "Cult: #{key[:name]}")
+        client.ssh_keys.create(dk_key).id
       end
 
 
@@ -80,39 +64,48 @@ module Cult
       end
 
 
-      def destroy!(id:)
+      def destroy!(id:, ssh_key_id: nil)
         client.droplets.delete(id: id)
+        destroy_ssh_key!(ssh_key_id: ssh_key_id) if ssh_key_id
       end
 
+      def destroy_ssh_key!(ssh_key_id:)
+        client.ssh_keys.delete(id: ssh_key_id)
+      end
 
-      def provision!(name:, size:, zone:, image:, ssh_key_files:)
-        fingerprints = Array(ssh_key_files).map do |file|
-          upload_ssh_key(file: file)[:fingerprint]
-        end
-
-        begin
-          params = {
-            name:     name,
-            size:     fetch_mapped(name: :size, from: sizes_map, key: size),
-            image:    fetch_mapped(name: :image, from: images_map, key: image),
-            region:   fetch_mapped(name: :zone, from: zones_map, key: zone),
-            ssh_keys: fingerprints,
-
-            private_networking: true,
-            ipv6: true
-          }
-        rescue KeyError => e
-          fail ArgumentError, "Invalid argument: #{e.message}"
-        end
-
-        droplet = DropletKit::Droplet.new(params)
+      def provision!(name:, size:, zone:, image:, ssh_public_key:)
+        transaction do |xac|
+          ssh_key_id = upload_ssh_key(file: ssh_public_key)
+          xac.rollback do
+            destroy_ssh_key!(id: ssh_key_id)
+          end
+
+          begin
+            params = {
+              name:     name,
+              size:     fetch_mapped(name: :size, from: sizes_map, key: size),
+              image:    fetch_mapped(name: :image, from: images_map, key: image),
+              region:   fetch_mapped(name: :zone, from: zones_map, key: zone),
+              ssh_keys: [ssh_key_id],
+
+              private_networking: true,
+              ipv6: true
+            }
+          rescue KeyError => e
+            fail ArgumentError, "Invalid argument: #{e.message}"
+          end
+
+          droplet = DropletKit::Droplet.new(params)
+
+          if droplet.nil?
+            fail "Droplet was nil: #{params.inspect}"
+          end
 
-        if droplet.nil?
-          fail "Droplet was nil: #{params.inspect}"
-        end
-
-        rollback_on_error(id: droplet.id) do
           droplet = client.droplets.create(droplet)
+          xac.rollback do
+            destroy!(id: droplet.id)
+          end
+
           droplet = await_creation(droplet)
 
           ipv4_public  = droplet.networks.v4.find {|n| n.type == 'public'  }
@@ -126,8 +119,8 @@ module Cult
               size:          size,
               zone:          zone,
               image:         image,
-              ssh_key_files: ssh_key_files,
-              ssh_keys:      fingerprints,
+
+              ssh_key_id:    ssh_key_id,
 
               id:            droplet.id,
               created_at:    droplet.created_at,

data/lib/cult/drivers/linode_driver.rb

@@ -1,6 +1,3 @@
-require 'cult/driver'
-require 'cult/cli/common'
-
 require 'securerandom'
 require 'time'
 
@@ -48,8 +45,6 @@ module Cult
     class LinodeDriver < ::Cult::Driver
       self.required_gems = 'linode'
 
-      include Common
-
       SWAP_SIZE = 256
 
       attr_reader :client
@@ -124,30 +119,29 @@ module Cult
       end
 
 
-      def destroy!(id:)
+      def destroy!(id:, ssh_key_id: [])
         client.linode.delete(linodeid: id, skipchecks: true)
       end
 
 
-      def provision!(name:, size:, zone:, image:, ssh_key_files:)
+      def provision!(name:, size:, zone:, image:, ssh_public_key:)
         sizeid  = fetch_mapped(name: :size, from: sizes_map, key: size)
         imageid = fetch_mapped(name: :image, from: images_map, key: image)
         zoneid  = fetch_mapped(name: :zone, from: zones_map, key: zone)
         disksize = disk_size_for_size(size)
 
-        linodeid = client.linode.create(datacenterid: zoneid,
-                                        planid: sizeid).linodeid
+        transaction do |xac|
+          linodeid = client.linode.create(datacenterid: zoneid,
+                                          planid: sizeid).linodeid
+          xac.rollback do
+            destroy!(id: linodeid)
+          end
 
-        rollback_on_error(id: linodeid) do
           # We give it a name early so we can find it in the Web UI if anything
           # goes wrong.
           client.linode.update(linodeid: linodeid, label: name)
           client.linode.ip.addprivate(linodeid: linodeid)
 
-          ssh_keys = Array(ssh_key_files).map do |file|
-            ssh_key_info(file: file)
-          end
-
           # You shouldn't run meaningful swap, but this makes the Web UI not
           # scare you, and apparently Linux runs better with ANY swap,
           # regardless of how small.  We've matched the small size the Linode
@@ -165,15 +159,15 @@ module Cult
             # Linode's max length is 128, generates longer than that to
             # no get the fixed == and truncates.
             rootpass: SecureRandom.base64(100)[0...128],
-            rootsshkey: ssh_keys.map {|k| k[:data] }.join("\n"),
+            rootsshkey: ssh_key_info(file: ssh_public_key)[:data],
             size: disksize - SWAP_SIZE
           }
 
           diskid = client.linode.disk.createfromdistribution(params).diskid
 
 
-          # We don't have to reference the config specifically: It'll be the only
-          # configuration that exists, so it'll be used.
+          # We don't have to reference the config specifically: It'll be the
+          # only configuration that exists, so it'll be used.
           client.linode.config.create(linodeid: linodeid,
                                       kernelid: latest_kernel_id,
                                       disklist: "#{diskid},#{swapid}",
@@ -200,8 +194,6 @@ module Cult
               size:          size,
               zone:          zone,
               image:         image,
-              ssh_key_files: ssh_keys.map{|k| k[:file]},
-              ssh_keys:      ssh_keys.map{|k| k[:fingerprint]},
 
               id:           linodeid,
               created_at:   Time.now.iso8601,
@@ -213,6 +205,7 @@ module Cult
               meta:         {}
           }
         end
+
       end
 
 

data/lib/cult/drivers/load.rb

@@ -1,6 +1,3 @@
-require 'cult/driver'
-require 'cult/named_array'
-
 module Cult
   module Drivers
 

data/lib/cult/drivers/vultr_driver.rb

@@ -1,6 +1,3 @@
-require 'cult/driver'
-require 'cult/drivers/common'
-
 require 'net/ssh'
 require 'time'
 
@@ -9,8 +6,6 @@ module Cult
     class VultrDriver < ::Cult::Driver
       self.required_gems = 'vultr'
 
-      include Common
-
       attr_reader :api_key
 
       def initialize(api_key:)
@@ -90,26 +85,11 @@ module Cult
       with_api_key    :sizes_map
 
 
-      def ssh_keys
-        Vultr::SSHKey.list[:result].values
-      end
-      memoize      :ssh_keys
-      with_api_key :ssh_keys
-
 
       def upload_ssh_key(file:)
         key = ssh_key_info(file: file)
-
-        vkey = if (exist = ssh_keys.find {|e| e["ssh_key"] == key[:data] })
-          exist
-        else
-          ssh_keys_dememo!
-          Vultr::SSHKey.create(name: "Cult: #{key[:name]}",
-                               ssh_key: key[:data])[:result]
-        end
-
-        vkey["fingerprint"] = key[:fingerprint]
-        vkey
+        Vultr::SSHKey.create(name: "Cult: #{key[:name]}",
+                             ssh_key: key[:data])[:result]["SSHKEYID"]
       end
       with_api_key :upload_ssh_key
 
@@ -121,34 +101,43 @@ module Cult
       end
 
 
-      def destroy!(id:)
+      def destroy!(id:, ssh_key_id: nil)
         Vultr::Server.destroy(SUBID: id)
+        destroy_ssh_key!(ssh_key_id: ssh_key_id) if ssh_key_id
       end
       with_api_key :destroy!
 
+      def destroy_ssh_key!(ssh_key_id:)
+        Vultr::SSHKey.destroy(SSHKEYID: ssh_key_id)
+      end
+      with_api_key :destroy_ssh_key!
 
-      def provision!(name:, size:, zone:, image:, ssh_key_files:)
-        keys = Array(ssh_key_files).map do |filename|
-          upload_ssh_key(file: filename)
-        end
-
-        sizeid  = fetch_mapped(name: :size, from: sizes_map, key: size)
-        imageid = fetch_mapped(name: :image, from: images_map, key: image)
-        zoneid  = fetch_mapped(name: :zone, from: zones_map, key: zone)
 
-        r = Vultr::Server.create(DCID: zoneid,
-                                 OSID: imageid,
-                                 VPSPLANID: sizeid,
-                                 enable_ipv6: 'yes',
-                                 enable_private_network: 'yes',
-                                 label: name,
-                                 hostname: name,
-                                 SSHKEYID: keys.map{|v| v["SSHKEYID"] }
-                                               .join(','))
+      def provision!(name:, size:, zone:, image:, ssh_public_key:)
+        transaction do |xac|
+          ssh_key_id = upload_ssh_key(file: ssh_public_key)
+          xac.rollback do
+            destroy_ssh_key!(ssh_key_id: ssh_key_id)
+          end
 
-        subid = r[:result]["SUBID"]
+          sizeid  = fetch_mapped(name: :size, from: sizes_map, key: size)
+          imageid = fetch_mapped(name: :image, from: images_map, key: image)
+          zoneid  = fetch_mapped(name: :zone, from: zones_map, key: zone)
+
+          r = Vultr::Server.create(DCID: zoneid,
+                                   OSID: imageid,
+                                   VPSPLANID: sizeid,
+                                   enable_ipv6: 'yes',
+                                   enable_private_network: 'yes',
+                                   label: name,
+                                   hostname: name,
+                                   SSHKEYID: ssh_key_id)
+
+          subid = r[:result]["SUBID"]
+          xac.rollback do
+            destroy!(id: subid)
+          end
 
-        rollback_on_error(id: subid) do
           # Wait until it's active, it won't have an IP until then
           backoff_loop do
             r = Vultr::Server.list(SUBID: subid)[:result]
@@ -166,8 +155,8 @@ module Cult
               size:          size,
               zone:          zone,
               image:         image,
-              ssh_key_files: ssh_key_files,
-              ssh_keys:      keys.map{|v| v["fingerprint"]},
+
+              ssh_key_id:    ssh_key_id,
 
               id:           subid,
               created_at:   Time.now.iso8601,