cult 0.1.1.pre

data/lib/cult/drivers/linode_driver.rb

@@ -0,0 +1,282 @@
+require 'cult/driver'
+require 'cult/cli/common'
+
+require 'securerandom'
+require 'time'
+
+module Cult
+  # This has been submitted as a PR.  It lets us set a label and custom
+  # expiration length for an API key.
+  #  See: https://github.com/rick/linode/pull/34
+  module LinodeMonkeyPatch
+    def fetch_api_key(options = {})
+      request = {
+        api_action: 'user.getapikey',
+        api_responseFormat: 'json',
+        username: username,
+        password: password
+      }
+
+      if options.key?(:label)
+        request[:label] = options[:label]
+      end
+
+      if options.key?(:expires)
+        expires = options[:expires]
+        request[:expires] = expires.nil? ? 0 : expires
+      end
+
+      response = post(request)
+      if error?(response)
+        fail "Errors completing request [user.getapikey] @ [#{api_url}] for " +
+             "username [#{username}]:\n" +
+             "#{error_message(response, 'user.getapikey')}"
+      end
+      reformat_response(response).api_key
+    end
+    public :fetch_api_key
+
+    module_function
+    def install!
+      ::Linode.prepend(self)
+    end
+  end
+end
+
+module Cult
+  module Drivers
+    class LinodeDriver < ::Cult::Driver
+      self.required_gems = 'linode'
+
+      include Common
+
+      SWAP_SIZE = 256
+
+      attr_reader :client
+
+      def initialize(api_key:)
+        LinodeMonkeyPatch.install!
+        @client = Linode.new(api_key: api_key)
+      end
+
+
+      def images_map
+        client.avail.distributions.select(&:is64bit).map do |v|
+          name = v.label
+          [ slugify(distro_name(v.label)), v.distributionid ]
+        end.to_h
+      end
+      memoize         :images_map
+      with_id_mapping :images_map
+
+
+      def zones_map
+        client.avail.datacenters.map do |v|
+          [ slugify(v.abbr), v.datacenterid ]
+        end.to_h
+      end
+      memoize         :zones_map
+      with_id_mapping :zones_map
+
+
+      def sizes_map
+        client.avail.linodeplans.map do |v|
+          name = v.label.gsub(/^Linode /, '')
+          if name.match(/^\d+$/)
+            mb = name.to_i
+            if mb < 1024
+              "#{mb}mb"
+            else
+              name = "#{mb / 1024}gb"
+            end
+          end
+          [ slugify(name), v.planid ]
+        end.to_h
+      end
+      memoize         :sizes_map
+      with_id_mapping :sizes_map
+
+
+      # We try to use the reasonable sizes that the web UI uses, although the
+      # API lets us change it.
+      def disk_size_for_size(size)
+        gb = 1024
+        {
+          '2gb'    => 24   * gb,
+          '4gb'    => 48   * gb,
+          '8gb'    => 96   * gb,
+          '12gb'   => 192  * gb,
+          '24gb'   => 384  * gb,
+          '48gb'   => 768  * gb,
+          '64gb'   => 1152 * gb,
+          '80gb'   => 1536 * gb,
+          '120gb'  => 1920 * gb
+        }.fetch(size.to_s)
+      end
+
+
+      # I've been told by Linode support that this literal will always mean
+      # "Latest x86".  But in case that changes...
+      def latest_kernel_id
+        @latest_kernel_id ||= 138 || begin
+          client.avail.kernels.find {|k| k.label.match(/^latest 64 bit/i)}
+        end.kernelid
+      end
+
+
+      def destroy!(id:)
+        client.linode.delete(linodeid: id, skipchecks: true)
+      end
+
+
+      def provision!(name:, size:, zone:, image:, ssh_key_files:)
+        sizeid  = fetch_mapped(name: :size, from: sizes_map, key: size)
+        imageid = fetch_mapped(name: :image, from: images_map, key: image)
+        zoneid  = fetch_mapped(name: :zone, from: zones_map, key: zone)
+        disksize = disk_size_for_size(size)
+
+        linodeid = client.linode.create(datacenterid: zoneid,
+                                        planid: sizeid).linodeid
+
+        rollback_on_error(id: linodeid) do
+          # We give it a name early so we can find it in the Web UI if anything
+          # goes wrong.
+          client.linode.update(linodeid: linodeid, label: name)
+          client.linode.ip.addprivate(linodeid: linodeid)
+
+          ssh_keys = Array(ssh_key_files).map do |file|
+            ssh_key_info(file: file)
+          end
+
+          # You shouldn't run meaningful swap, but this makes the Web UI not
+          # scare you, and apparently Linux runs better with ANY swap,
+          # regardless of how small.  We've matched the small size the Linode
+          # Web UI does by default.
+          swapid = client.linode.disk.create(linodeid: linodeid,
+                                             label: "Cult: #{name}-swap",
+                                             type: "swap",
+                                             size: SWAP_SIZE).diskid
+
+          # Here, we create the OS on-node storage
+          params = {
+            linodeid: linodeid,
+            distributionid: imageid,
+            label: "Cult: #{name}",
+            # Linode's max length is 128, generates longer than that to
+            # no get the fixed == and truncates.
+            rootpass: SecureRandom.base64(100)[0...128],
+            rootsshkey: ssh_keys.map {|k| k[:data] }.join("\n"),
+            size: disksize - SWAP_SIZE
+          }
+
+          diskid = client.linode.disk.createfromdistribution(params).diskid
+
+
+          # We don't have to reference the config specifically: It'll be the only
+          # configuration that exists, so it'll be used.
+          client.linode.config.create(linodeid: linodeid,
+                                      kernelid: latest_kernel_id,
+                                      disklist: "#{diskid},#{swapid}",
+                                      rootdevicenum: 1,
+                                      label: "Cult: Latest Linux-x64")
+
+          client.linode.reboot(linodeid: linodeid)
+
+          # Information gathering step...
+          all_ips = client.linode.ip.list(linodeid: linodeid)
+
+          ipv4_public  = all_ips.find{ |ip| ip.ispublic == 1 }&.ipaddress
+          ipv4_private = all_ips.find{ |ip| ip.ispublic == 0 }&.ipaddress
+
+          # This is a shame: Linode has awesome support for ipv6, but doesn't
+          # expose it in the API.
+          ipv6_public  = nil
+          ipv6_private = nil
+
+          await_ssh(ipv4_public)
+
+          return {
+              name:          name,
+              size:          size,
+              zone:          zone,
+              image:         image,
+              ssh_key_files: ssh_keys.map{|k| k[:file]},
+              ssh_keys:      ssh_keys.map{|k| k[:fingerprint]},
+
+              id:           linodeid,
+              created_at:   Time.now.iso8601,
+              host:         ipv4_public,
+              ipv4_public:  ipv4_public,
+              ipv4_private: ipv4_private,
+              ipv6_public:  ipv6_public,
+              ipv6_private: ipv6_private,
+              meta:         {}
+          }
+        end
+      end
+
+
+      def self.interrupts
+        # I hate IRB.
+        [Interrupt] + (defined?(IRB) ? [IRB::Abort] : [])
+      end
+
+
+      def self.setup!
+        super
+        LinodeMonkeyPatch.install!
+
+        linode = nil
+        api_key = nil
+
+        begin
+          loop do
+            puts "Cult needs an API key.  It can get one for you, but will " +
+                 "need your Linode username and password.  If you'd rather "
+                 "generate it at Linode, hit ctrl-c"
+            username = CLI.ask "Username"
+            password = CLI.password "Password"
+            linode = Linode.new(username: username, password: password)
+            begin
+              linode.fetch_api_key(label: "Cult", expires: nil)
+              api_key = linode.api_key
+              fail RuntimeError if api_key.nil?
+              puts "Got it!  In case you're curious: #{api_key}"
+            rescue RuntimeError
+              puts "Linode disagreed with your password."
+              next if CLI.yes_no?("Try again?")
+            end
+            break
+          end
+        rescue *interrupts
+          puts
+          url = "https://manager.linode.com/profile/api"
+          puts "You can obtain an API key for Cult at the following URL:"
+          puts "  #{url}"
+          puts
+          CLI.launch_browser(url) if CLI.yes_no?("Open Browser?")
+          api_key = CLI.prompt("API Key")
+        end
+
+        linode ||= Linode.new(api_key: api_key)
+        resp = linode.test.echo(message: "PING")
+        if resp.message != 'PING'
+          raise "Didn't respond to ping.  Something went wrong."
+        end
+
+        inst = new(api_key: api_key)
+
+        return {
+          api_key: api_key,
+          driver: driver_name,
+          configurations: {
+            sizes:  inst.sizes,
+            zones:  inst.zones,
+            images: inst.images,
+          }
+        }
+      end
+
+    end
+  end
+end

data/lib/cult/drivers/load.rb

@@ -0,0 +1,26 @@
+require 'cult/driver'
+require 'cult/named_array'
+
+module Cult
+  module Drivers
+
+    module_function
+    def load!
+      Dir.glob(File.join(__dir__, "*_driver.rb")).each do |file|
+        require file
+      end
+    end
+
+
+    def all
+      Cult::Drivers.constants(false).map do |m|
+        Cult::Drivers.const_get(m)
+      end.select do |cls|
+        ::Cult::Driver > cls
+      end.to_named_array
+    end
+
+  end
+end
+
+Cult::Drivers.load!

data/lib/cult/drivers/script_driver.rb

@@ -0,0 +1,27 @@
+require 'cult/driver'
+require 'cult/drivers/common'
+
+module Cult
+  module Drivers
+
+    class ScriptDriver < ::Cult::Driver
+      include Common
+
+      def initialize(api_key:)
+        fail NotImplementedError
+      end
+
+
+      def provision!(name:, size:, zone:, image:, ssh_key_files:, extra: {})
+        fail NotImplementedError
+      end
+
+
+      def self.setup!
+        super
+        fail NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/cult/drivers/vultr_driver.rb

@@ -0,0 +1,217 @@
+require 'cult/driver'
+require 'cult/drivers/common'
+
+require 'net/ssh'
+require 'time'
+
+module Cult
+  module Drivers
+    class VultrDriver < ::Cult::Driver
+      self.required_gems = 'vultr'
+
+      include Common
+
+      attr_reader :api_key
+
+      def initialize(api_key:)
+        @api_key = api_key
+      end
+
+
+      # This sets the Vultr API key to this instance's api key for the duration
+      # of a method call and restores it afterwards.
+      def self.with_api_key(method_name)
+        unwrapped_name = "#{method_name}_no_api_key".to_sym
+        alias_method unwrapped_name, method_name
+        define_method(method_name) do |*args, &block|
+          old_api_key = Vultr.api_key
+          begin
+            Vultr.api_key = self.api_key
+            return send(unwrapped_name, *args, &block)
+          ensure
+            Vultr.api_key = old_api_key
+          end
+        end
+      end
+
+
+      def zones_map
+        Vultr::Region.list[:result].map do |k, v|
+          [slugify(v["regioncode"]), v["DCID"]]
+        end.to_h
+      end
+      memoize         :zones_map
+      with_id_mapping :zones_map
+      with_api_key    :zones_map
+
+
+      def images_map
+        Vultr::OS.list[:result].select do |k, v|
+          # Doing our part to kill x86/32
+          v['arch'] == 'x64'
+        end.map do |k,v|
+          [slugify(distro_name(v["name"])), v["OSID"]]
+        end.reject do |k,v|
+          %w(custom snapshot backup application).include?(k) ||
+          k.match(/^windows/)
+        end.to_h
+      end
+      memoize         :images_map
+      with_id_mapping :images_map
+      with_api_key    :images_map
+
+
+      def sizes_map
+        Vultr::Plan.list[:result].values.select do |v|
+          v["plan_type"] == 'SSD'
+        end.map do |v|
+          if (m = v["name"].match(/^(\d+) ([MGTP]B) RAM/i))
+            _, ram, unit = *m
+            ram = ram.to_i
+
+            if unit == "MB" && ram >= 1024
+              ram = ram / 1024
+              unit = "GB"
+            end
+
+            if unit == "GB" && ram >= 1024
+              ram = ram / 1024
+              unit = "TB"
+            end
+
+            ["#{ram}#{unit}".downcase, v["VPSPLANID"] ]
+          else
+            nil
+          end
+        end.compact.to_h
+      end
+      memoize         :sizes_map
+      with_id_mapping :sizes_map
+      with_api_key    :sizes_map
+
+
+      def ssh_keys
+        Vultr::SSHKey.list[:result].values
+      end
+      memoize      :ssh_keys
+      with_api_key :ssh_keys
+
+
+      def upload_ssh_key(file:)
+        key = ssh_key_info(file: file)
+
+        vkey = if (exist = ssh_keys.find {|e| e["ssh_key"] == key[:data] })
+          exist
+        else
+          ssh_keys_dememo!
+          Vultr::SSHKey.create(name: "Cult: #{key[:name]}",
+                               ssh_key: key[:data])[:result]
+        end
+
+        vkey["fingerprint"] = key[:fingerprint]
+        vkey
+      end
+      with_api_key :upload_ssh_key
+
+
+      def fetch_ip(list, type)
+        goal = (type == :public ? "main_ip" : "private")
+        r = list.find{ |v| v["type"] == goal }
+        r.nil? ? nil : r["ip"]
+      end
+
+
+      def destroy!(id:)
+        Vultr::Server.destroy(SUBID: id)
+      end
+      with_api_key :destroy!
+
+
+      def provision!(name:, size:, zone:, image:, ssh_key_files:)
+        keys = Array(ssh_key_files).map do |filename|
+          upload_ssh_key(file: filename)
+        end
+
+        sizeid  = fetch_mapped(name: :size, from: sizes_map, key: size)
+        imageid = fetch_mapped(name: :image, from: images_map, key: image)
+        zoneid  = fetch_mapped(name: :zone, from: zones_map, key: zone)
+
+        r = Vultr::Server.create(DCID: zoneid,
+                                 OSID: imageid,
+                                 VPSPLANID: sizeid,
+                                 enable_ipv6: 'yes',
+                                 enable_private_network: 'yes',
+                                 label: name,
+                                 hostname: name,
+                                 SSHKEYID: keys.map{|v| v["SSHKEYID"] }
+                                               .join(','))
+
+        subid = r[:result]["SUBID"]
+
+        rollback_on_error(id: subid) do
+          # Wait until it's active, it won't have an IP until then
+          backoff_loop do
+            r = Vultr::Server.list(SUBID: subid)[:result]
+            throw :done if r['status'] == 'active'
+          end
+
+          iplist4 = Vultr::Server.list_ipv4(SUBID: subid)[:result].values[0]
+          iplist6 = Vultr::Server.list_ipv6(SUBID: subid)[:result].values[0]
+
+          host = fetch_ip(iplist4, :public)
+          await_ssh(host)
+
+          return {
+              name:          name,
+              size:          size,
+              zone:          zone,
+              image:         image,
+              ssh_key_files: ssh_key_files,
+              ssh_keys:      keys.map{|v| v["fingerprint"]},
+
+              id:           subid,
+              created_at:   Time.now.iso8601,
+              host:         host,
+              ipv4_public:  host,
+              ipv4_private: fetch_ip(iplist4, :private),
+              ipv6_public:  fetch_ip(iplist6, :public),
+              ipv6_private: fetch_ip(iplist6, :private),
+              meta:         {}
+          }
+        end
+      end
+      with_api_key :provision!
+
+
+      def self.setup!
+        super
+        url = "https://my.vultr.com/settings/#settingsapi"
+        puts "Vultr does not generate multiple API keys, so you'll need to "
+             "create one (if it does not exist).  You can access your API key "
+             "at the following URL:"
+        puts
+        puts "  #{url}"
+        puts
+
+        CLI.launch_browser(url) if CLI.yes_no?("Launch browser?")
+
+        api_key = CLI.prompt("API Key")
+
+        unless api_key.match(/^[A-Z2-7]{36}$/)
+          puts "That doesn't look like an API key, but I'll trust you"
+        end
+
+        inst = new(api_key: api_key)
+        return {
+          api_key: api_key,
+          driver: driver_name,
+          configurations: {
+            sizes:  inst.sizes,
+            zones:  inst.zones,
+            images: inst.images,
+          }
+        }
+      end
+    end
+  end
+end

data/lib/cult/named_array.rb

@@ -0,0 +1,129 @@
+# A lot of times, we want a sequential array of objects, but it'd still be
+# really convenient to refer to things by their name.  This is particularly
+# painful in the console, where, e.g., nodes can only be referred to by index,
+# and you end up calling `find` a lot.
+#
+# NamedArray is an array, but overloads [] to also work with a String, Symbol,
+# Regexp, or a few other things. e.g., nodes[:something].  It works by finding
+# the first item who responds from `named_array_identifier` with the matching
+# key.
+#
+# By default named_array_identifier returns name, but this can be overridden.
+
+module Cult
+  class NamedArray < Array
+    # Any Array can convert itself to a NamedArray
+    module ArrayExtensions
+      def to_named_array
+        NamedArray.new(self)
+      end
+      ::Array.include(self)
+    end
+
+
+    # This maps #named_array_identifier to #name by default
+    module ObjectExtensions
+      def named_array_identifier
+        name
+      end
+      ::Object.include(self)
+    end
+
+
+    def to_named_array
+      self
+    end
+
+
+    # Wrap any non-mutating methods that can return an Array,
+    # and wrap the result with a NamedArray.  This is why NamedArray.select
+    # results in a NamedArray instead of an Array
+    PROXY_METHODS = %i(& * + - << | collect compact flatten reject reverse
+                       rotate select shuffle slice sort uniq)
+    PROXY_METHODS.each do |method_name|
+      define_method(method_name) do |*args, &b|
+        r = super(*args, &b)
+        r.respond_to?(:to_named_array) ? r.to_named_array : r
+      end
+    end
+
+
+    # It's unforunate that there's not a Regexp constructor that'll
+    # accept this string format with options.
+    def build_regexp_from_string(s)
+      fail RegexpError, "Isn't a Regexp: #{s}" if s[0] != '/'
+      options = extract_regexp_options(s)
+      Regexp.new(s[1 ... s.rindex('/')], options)
+    end
+    private :build_regexp_from_string
+
+
+    def extract_regexp_options(s)
+      offset = s.rindex('/')
+      fail RegexpError, "Unterminated Regexp: #{s}" if offset == 0
+
+      trailing = s[offset + 1 ... s.size]
+      re_string = "%r!!#{trailing}"
+      begin
+        (eval re_string).options
+      rescue SyntaxError => e
+        fail RegexpError, "invalid Regexp options: #{trailing}"
+      end
+    end
+    private :extract_regexp_options
+
+
+    # Returns all keys that match if method == :select, the first if
+    # method == :find
+    def all(key, method = :select)
+      key = case key
+        when Integer
+          # Fallback to default behavior
+          return super
+        when String
+          key[0] == '/' ? build_regexp_from_string(key) : key
+        when Regexp, Proc, Range
+          key
+        when Symbol
+          key.to_s
+        when NilClass
+          return nil
+        else
+          fail KeyError, "#{key} did not resolve to an object"
+      end
+
+      send(method) do |v|
+        key === v.named_array_identifier
+      end
+    end
+
+
+    # first matching item
+    def [](key)
+      return super if key.is_a?(Integer)
+      all(key, :find)
+    end
+
+
+    # first matching item, or raises KeyError
+    def fetch(key)
+      all(key, :find) or raise KeyError
+    end
+
+
+    def key?(key)
+      !! all(key, :find)
+    end
+    alias_method :exist?, :key?
+
+
+    def keys
+      map(&:named_array_identifier)
+    end
+
+
+    def values
+      self
+    end
+  end
+end