cult 0.1.1.pre

data/lib/cult/commander.rb

@@ -0,0 +1,103 @@
+require 'net/ssh'
+require 'net/scp'
+require 'shellwords'
+require 'rainbow'
+require 'rubygems/package'
+require 'rubygems/package/tar_writer'
+
+module Cult
+
+  class Bundle
+    attr_reader :tar
+    def initialize(io, &block)
+      @tar = Gem::Package::TarWriter.new(io)
+      if block_given?
+        begin
+          yield self
+        ensure
+          @tar.close
+          @tar = nil
+        end
+      end
+    end
+
+    def add_file(project, role, node, transferable)
+      data = transferable.contents(project, role, node, pwd: role.path)
+      tar.add_file(transferable.remote_path, transferable.file_mode) do |io|
+        io.write(data)
+      end
+    end
+  end
+
+  class Commander
+    attr_reader :project
+    attr_reader :node
+
+    def initialize(project:, node:)
+      @project = project
+      @node = node
+    end
+
+    def esc(s)
+      Shellwords.escape(s)
+    end
+
+    def send_bundle(ssh, role)
+      io = StringIO.new
+      Bundle.new(io) do |bundle|
+        puts "Building bundle..."
+        role.build_order.each do |r|
+          (r.artifacts + r.tasks).each do |transferable|
+            bundle.add_file(project, r, node, transferable)
+          end
+        end
+      end
+      filename = "cult-#{role.name}.tar"
+      puts "Uploading bundle #{filename}..."
+
+      scp = Net::SCP.new(ssh)
+      io.rewind
+      scp.upload!(io, filename)
+      ssh.exec! "tar -xf #{esc(filename)} && rm #{esc(filename)}"
+    end
+
+    def install!(role)
+      connect(user: role.definition['user']) do |ssh|
+        send_bundle(ssh, role)
+
+        role.build_order.each do |r|
+          puts "Installing role: #{Rainbow(r.name).blue}"
+          working_dir = r.remote_path
+          r.tasks.each do |t|
+            puts "Executing: #{t.remote_path}"
+            task_bin = r.relative_path(t.path)
+            res = ssh.exec! <<~BASH
+              cd #{esc(working_dir)}; \
+                if [ ! -f ./#{esc(task_bin)}.success ]; then  \
+                  touch ./#{esc(task_bin)}.attempt && \
+                  ./#{esc(task_bin)} && \
+                  mv ./#{esc(task_bin)}.attempt ./#{esc(task_bin)}.success; \
+                fi
+            BASH
+            unless res.empty?
+              puts Rainbow(res.gsub(/^/, '    ')).darkgray.italic
+            end
+          end
+        end
+      end
+    end
+
+    def bootstrap!
+      bootstrap_role = CLI.fetch_item('bootstrap', from: Role)
+      install!(bootstrap_role)
+    end
+
+    def connect(user:, &block)
+      puts "Connecting with user=#{user}"
+      Net::SSH.start(node.host, user) do |ssh|
+        yield ssh
+      end
+    end
+  end
+
+end

data/lib/cult/config.rb

@@ -0,0 +1,22 @@
+module Cult
+  module_function
+  def project=(project)
+    @project = project
+  end
+
+  def project
+    @project
+  end
+
+  # This is a a mode we're considering: when it's set, certain objects
+  # will be frozen when they're created, and creating a "logically the same"
+  # instance (e.g., role with same name in the same project) will return
+  # the actual same object each time.
+  #
+  # I'm not sure if we'll commit to this, but having this toggle lets us
+  # easily see what breaks.
+  def immutable?
+    ENV['CULT_IMMUTABLE'] == '1'
+  end
+
+end

data/lib/cult/definition.rb

@@ -0,0 +1,112 @@
+require 'yaml'
+require 'json'
+require 'forwardable'
+
+require 'cult/template'
+
+module Cult
+  class Definition
+    attr_reader :object
+    attr_reader :bag
+
+    extend Forwardable
+    def_delegators :object, :definition_parameters, :definition_path,
+                            :definition_parents
+
+
+    def initialize(object)
+      @object = object
+    end
+
+
+    def inspect
+      "\#<#{self.class.name} " +
+        "object: #{object.inspect}, " +
+        "params: #{definition_parameters}, " +
+        "parents: #{definition_parents}, " +
+        "direct_values: #{bag}>"
+    end
+    alias_method :to_s, :inspect
+
+
+    def filenames
+      Array(definition_path).map do |dp|
+        attempt = [ "#{dp}.yaml", "#{dp}.yml", "#{dp}.json" ]
+        existing = attempt.select do |filename|
+          File.exist?(filename)
+        end
+        if existing.size > 1
+          raise RuntimeError, "conflicting definition files: #{existing}"
+        end
+        existing[0]
+      end.compact
+    end
+
+
+    def decoder_for(filename)
+      @decoder_for ||= begin
+        case filename
+          when nil
+            nil
+          when /\.json\z/
+            JSON.method(:parse)
+          when /\.ya?ml\z/
+            YAML.method(:safe_load)
+          else
+            fail RuntimeError, "No decoder for file type: #{filename}"
+        end
+      end
+    end
+
+
+    def bag
+      @bag ||= begin
+        result = {}
+        filenames.each do |filename|
+          erb = Template.new(definition_parameters)
+          contents = erb.process(File.read(filename))
+          result.merge! decoder_for(filename).call(contents)
+        end
+        result
+      end
+    end
+    alias_method :to_h, :bag
+
+
+    def direct(k)
+      fail "Use string keys" unless k.is_a?(String)
+      bag[k]
+    end
+
+
+    def [](k)
+      fail "Use string keys" unless k.is_a?(String)
+      if bag.key?(k)
+        bag[k]
+      else
+        parent_responses = definition_parents.map do |p|
+          [p, p.definition[k]]
+        end.reject do |k, v|
+          v.nil?
+        end
+        consensus = parent_responses.group_by(&:last)
+        if consensus.empty?
+          return nil
+        elsif consensus.size != 1
+          msg = "#{object.inspect}: I didn't have key '#{k}', and " +
+                "my parents had conflicting answers: " +
+                "[answer, parents]: #{consensus}"
+          fail KeyError, msg
+        end
+        consensus.keys[0]
+      end
+    end
+
+
+    def []=(k, v)
+      fail "Use string keys" unless k.is_a?(String)
+      bag[k] = v
+    end
+
+  end
+end

data/lib/cult/driver.rb

@@ -0,0 +1,88 @@
+require 'cult/definition'
+require 'cult/drivers/common'
+
+module Cult
+  class Driver
+
+    # This is raised when a Driver is instantiated,  but the required
+    # gems are not installed.
+    class GemNeededError < RuntimeError
+      attr_reader :gems
+      def initialize(gems)
+        @gems = gems
+        super(message)
+      end
+
+      def message
+        "gems required: #{gems.inspect}"
+      end
+    end
+
+
+    class << self
+      attr_accessor :required_gems
+
+      def driver_name
+        name.split('::')
+            .last
+            .sub(/Driver\z/, '')
+            .gsub(/([a-z])([A-Z])/, '\1-\2')
+            .downcase
+      end
+
+
+      def inspect
+        self == Driver ? super : "#{super}/#{driver_name}"
+      end
+      alias_method :to_s, :inspect
+
+
+      def named_array_identifier
+        driver_name
+      end
+    end
+
+    def inspect
+      "\#<#{self.class.name} \"#{self.class.driver_name}\">"
+    end
+
+
+    def to_s
+      self.class.driver_name
+    end
+
+
+    # Attempts to loads all of the required gems before doing any real work
+    def self.try_requires!
+      req = Array(required_gems).map do |gem|
+        begin
+          require gem
+          nil
+        rescue LoadError
+          gem
+        end
+      end.compact
+
+      unless req.empty?
+        fail GemNeededError.new(req)
+      end
+    end
+
+
+    # These are helpers that most implementations will need, but Driver itself
+    # doesn't depend on.  Things like exponential back-off, awaiting an SSH
+    # port to open, etc.
+    include ::Cult::Drivers::Common
+
+
+    def self.setup!
+      try_requires!
+    end
+
+
+    def self.new(api_key:)
+      try_requires!
+      super
+    end
+  end
+end

data/lib/cult/drivers/common.rb

@@ -0,0 +1,192 @@
+require 'socket'
+require 'net/ssh'
+
+module Cult
+  module Drivers
+
+    module Common
+      module ClassMethods
+        # Lets us write a method "something_map" that returns {'ident' => ...},
+        # and also get a function "something" that returns the keys.
+        def with_id_mapping(method_name)
+          new_method = method_name.to_s.sub(/_map\z/, '')
+          define_method(new_method) do
+            send(method_name).keys
+          end
+        end
+
+
+        def memoize(method_name)
+          old_method_name = "#{method_name}_unmemoized".to_sym
+
+          alias_method old_method_name, method_name
+
+          var_name = "@#{method_name}".to_sym
+
+          define_method(method_name) do
+            if !instance_variable_defined?(var_name)
+              instance_variable_set(var_name, send(old_method_name))
+            end
+            instance_variable_get(var_name)
+          end
+
+          define_method("#{method_name}_unmemo!") do
+            remove_instance_variable(var_name)
+          end
+        end
+      end
+
+
+      def self.included(cls)
+        cls.extend(ClassMethods)
+      end
+
+
+      # works with with_id_mapping to convert a human-readible/normalized key
+      # to the id the backend service expects.  Allows '=value' to force a
+      # literal value, and gives better error messages.
+      def fetch_mapped(name:, from:, key:)
+        # Allow for the override.
+        key = key.to_s
+        return key[1..-1] if key[0] == '='
+
+        begin
+          from.fetch(key)
+        rescue KeyError => e
+          raise ArgumentError, "Invalid #{name}: \"#{key}\".  " +
+                               "Use \"=#{key}\" to force, or use one of: " +
+                               from.keys.inspect
+        end
+      end
+
+
+      def ssh_key_info(data: nil, file: nil)
+        if data.nil?
+          fail ArgumentError if file.nil?
+          data = File.read(file)
+        else
+          fail ArgumentError unless file.nil?
+        end
+
+        data = data.chomp
+        key = Net::SSH::KeyFactory.load_data_public_key(data, file)
+
+        fields = data.split(/ /)
+        return {
+          name:        fields[-1],
+          fingerprint: key.fingerprint,
+          data:        data,
+          file:        file
+        }
+      end
+
+
+      # Enter this block once a node has been created.  It makes sure it's
+      # destroyed if there's an error later in the procedure.
+      def rollback_on_error(id:, &block)
+        begin
+          yield
+        rescue Exception => e
+          begin
+            destroy!(id: id)
+          ensure
+            raise e
+          end
+        end
+      end
+
+
+      def slugify(s)
+        s.gsub(/[^a-z0-9]+/i, '-').gsub(/(^\-)|(-\z)/, '').downcase
+      end
+
+
+      def distro_name(s)
+        s = s.gsub(/\bx64\b/i, '')
+        # People sometimes add "LTS" to the name of Ubuntu LTS releases
+        s = s.gsub(/\blts\b/i, '') if s.match(/ubuntu/i)
+
+        # We don't particularly need the debian codename
+        s = s.gsub(/(\d)[\s-]+(\S+)/, '\1') if s.match(/^debian/i)
+        s
+      end
+
+
+      # Does back-off retrying.  Defaults to not-exponential.
+      # Block must throw :done to signal they are done.
+      def backoff_loop(wait = 3, scale = 1.2, &block)
+        times = 0
+        total_wait = 0.0
+
+        catch :done do
+          loop do
+            yield times, total_wait
+            sleep wait
+            times += 1
+            total_wait += wait
+            wait *= scale
+          end
+        end
+      end
+
+
+      # Waits until SSH is available at host.  "available" jsut means
+      # "listening"/acceping connections.
+      def await_ssh(host)
+        backoff_loop do
+          begin
+            sock = connect_timeout(host, 22, 1)
+            throw :done
+          rescue Errno::ETIMEDOUT, Errno::ECONNREFUSED
+            # Nothing, these are expected
+          ensure
+            sock.close if sock
+          end
+        end
+      end
+
+
+      # This should not be needed, but it is:
+      # https://spin.atomicobject.com/2013/09/30/socket-connection-timeout-ruby/
+      def connect_timeout(host, port, timeout = 5)
+        # Convert the passed host into structures the non-blocking calls
+        # can deal with
+        addr = Socket.getaddrinfo(host, nil)
+        sockaddr = Socket.pack_sockaddr_in(port, addr[0][3])
+
+        Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0).tap do |socket|
+          socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+
+          begin
+            # Initiate the socket connection in the background. If it doesn't
+            # fail immediately it will raise an IO::WaitWritable
+            # (Errno::EINPROGRESS) indicating the connection is in progress.
+            socket.connect_nonblock(sockaddr)
+
+          rescue IO::WaitWritable
+            # IO.select will block until the socket is writable or the timeout
+            # is exceeded - whichever comes first.
+            if IO.select(nil, [socket], nil, timeout)
+              begin
+                # Verify there is now a good connection
+                socket.connect_nonblock(sockaddr)
+              rescue Errno::EISCONN
+                # Good news everybody, the socket is connected!
+              rescue
+                # An unexpected exception was raised - the connection is no good.
+                socket.close
+                raise
+              end
+            else
+              # IO.select returns nil when the socket is not ready before
+              # timeout seconds have elapsed
+              socket.close
+              raise Errno::ETIMEDOUT
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/cult/drivers/digital_ocean_driver.rb

@@ -0,0 +1,179 @@
+require 'cult/driver'
+require 'cult/drivers/common'
+require 'net/ssh'
+require 'json'
+
+module Cult
+  module Drivers
+
+    class DigitalOceanDriver < ::Cult::Driver
+      self.required_gems = 'droplet_kit'
+
+      include Common
+
+      attr_reader :client
+
+      def initialize(api_key:)
+        @client = DropletKit::Client.new(access_token: api_key)
+      end
+
+
+      def sizes_map
+        client.sizes.all.to_a.map do |s|
+          [s.slug, s.slug]
+        end.to_h
+      end
+      memoize :sizes_map
+      with_id_mapping :sizes_map
+
+
+      def images_map
+        distros = %w(ubuntu coreos centos freebsd fedora debian).join '|'
+        re = /^(#{distros})\-.*\-x64$/
+        client.images.all.to_a.select do |image|
+          image.public && image.slug && image.slug.match(re)
+        end.map do |image|
+          [slugify(distro_name(image.slug)), image.slug]
+        end.to_h
+      end
+      memoize :images_map
+      with_id_mapping :images_map
+
+
+      def zones_map
+        client.regions.all.map do |zone|
+          [zone.slug, zone.slug]
+        end.to_h
+      end
+      memoize :zones_map
+      with_id_mapping :zones_map
+
+
+      def ssh_keys
+        client.ssh_keys.all.to_a.map(&:to_h)
+      end
+      memoize :ssh_keys
+
+
+      def upload_ssh_key(file:)
+        key = ssh_key_info(file: file)
+        # If we already have one with this fingerprint, use it.
+        if (exist = ssh_keys.find {|dk| dk[:fingerprint] == key[:fingerprint]})
+          exist
+        else
+          ssh_keys_dememo!
+          client.ssh_keys.create \
+            DropletKit::SSHKey.new(fingerprint: key[:fingerprint],
+                                   public_key: key[:data],
+                                   name: key[:name])
+        end
+      end
+
+
+      def await_creation(droplet)
+        d = nil
+        backoff_loop do
+          d = client.droplets.find(id: droplet.id)
+          throw :done if d.status == 'active'
+        end
+        return d
+      end
+
+
+      def destroy!(id:)
+        client.droplets.delete(id: id)
+      end
+
+
+      def provision!(name:, size:, zone:, image:, ssh_key_files:)
+        fingerprints = Array(ssh_key_files).map do |file|
+          upload_ssh_key(file: file)[:fingerprint]
+        end
+
+        begin
+          params = {
+            name:     name,
+            size:     fetch_mapped(name: :size, from: sizes_map, key: size),
+            image:    fetch_mapped(name: :image, from: images_map, key: image),
+            region:   fetch_mapped(name: :zone, from: zones_map, key: zone),
+            ssh_keys: fingerprints,
+
+            private_networking: true,
+            ipv6: true
+          }
+        rescue KeyError => e
+          fail ArgumentError, "Invalid argument: #{e.message}"
+        end
+
+        droplet = DropletKit::Droplet.new(params)
+
+        if droplet.nil?
+          fail "Droplet was nil: #{params.inspect}"
+        end
+
+        rollback_on_error(id: droplet.id) do
+          droplet = client.droplets.create(droplet)
+          droplet = await_creation(droplet)
+
+          ipv4_public  = droplet.networks.v4.find {|n| n.type == 'public'  }
+          ipv4_private = droplet.networks.v4.find {|n| n.type == 'private' }
+          ipv6_public  = droplet.networks.v6.find {|n| n.type == 'public'  }
+          ipv6_private = droplet.networks.v6.find {|n| n.type == 'private' }
+
+          await_ssh(ipv4_public.ip_address)
+          return {
+              name:          droplet.name,
+              size:          size,
+              zone:          zone,
+              image:         image,
+              ssh_key_files: ssh_key_files,
+              ssh_keys:      fingerprints,
+
+              id:            droplet.id,
+              created_at:    droplet.created_at,
+              host:          ipv4_public&.ip_address,
+              ipv4_public:   ipv4_public&.ip_address,
+              ipv4_private:  ipv4_private&.ip_address,
+              ipv6_public:   ipv6_public&.ip_address,
+              ipv6_private:  ipv6_private&.ip_address,
+              # Get rid of magic in droplet.
+              meta:          JSON.parse(droplet.to_json)
+          }
+        end
+      end
+
+
+      def self.setup!
+        super
+        url = "https://cloud.digitalocean.com/settings/api/tokens/new"
+
+        puts "Cult needs a read/write Access Token created for your " +
+             "DigitalOcean account."
+        puts "One can be generated at the following URL:"
+        puts
+        puts "  #{url}"
+        puts
+
+        CLI.launch_browser(url) if CLI.yes_no?("Open Browser?")
+
+        api_key = CLI.prompt("Access Token")
+        unless api_key.match(/\A[0-9a-f]{64}\z/)
+          puts "That doesn't look like an access token, but we'll take it."
+        end
+
+        inst = new(api_key: api_key)
+
+        return {
+          api_key: api_key,
+          driver: driver_name,
+          configurations: {
+            sizes:  inst.sizes,
+            zones:  inst.zones,
+            images: inst.images,
+          }
+        }
+      end
+    end
+
+  end
+end