cult 0.1.3.pre → 0.1.4.pre

data/lib/cult/named_array.rb

@@ -24,11 +24,48 @@ module Cult
     # This maps #named_array_identifier to #name by default
     module ObjectExtensions
       def named_array_identifier
-        name
+        respond_to?(:name) ? name : nil
       end
       ::Object.include(self)
     end
 
+    # Allows named_array.all[/something/]
+    class IndexWrapper
+      def initialize(ary, method_name)
+        @ary, @method_name = ary, method_name
+      end
+
+      def inspect
+        "\#<#{self.class.name}>"
+      end
+
+      def [](*args)
+        @ary.send(@method_name, *args)
+      end
+
+      def to_a
+        @ary
+      end
+      alias_method :to_ary, :to_a
+
+      def to_named_array
+        @ary.to_named_array
+      end
+    end
+    private_constant :IndexWrapper
+
+    def self.indexable_wrapper(method_name)
+      old_method_name = "#{method_name}_without_wrapper"
+      alias_method old_method_name, method_name
+      define_method(method_name) do |*a|
+        if a.empty?
+          return IndexWrapper.new(self, method_name)
+        else
+          return send(old_method_name, *a)
+        end
+      end
+    end
+
 
     def to_named_array
       self
@@ -39,7 +76,7 @@ module Cult
     # and wrap the result with a NamedArray.  This is why NamedArray.select
     # results in a NamedArray instead of an Array
     PROXY_METHODS = %i(& * + - << | collect compact flatten reject reverse
-                       rotate select shuffle slice sort uniq)
+                       rotate select shuffle slice sort uniq sort_by)
     PROXY_METHODS.each do |method_name|
       define_method(method_name) do |*args, &b|
         r = super(*args, &b)
@@ -87,41 +124,85 @@ module Cult
         when NilClass
           nil
         else
-          fail KeyError, "Invalid predicate: #{predicate.inspect}"
+          predicate
       end
     end
     private :expand_predicate
 
+    def extract_index(key)
+      re = /\[\s*([^\]]*)\s*\]$/
+      if key.is_a?(String) && (m = key.match(re))
+        subs, expr = m[0], m[1]
+        index = case expr
+          when /^(\-?\d+)$/; $1.to_i #.. $1.to_i
+          when /^(\-?\d+)\s*\.\.\s*(\-?\d+)$/; $1.to_i .. $2.to_i
+          when /^(\-?\d+)\s*\.\.\.\s*(\-?\d+)$/; $1.to_i ... $2.to_i
+          when /^((?:\-?\d+\s*,?\s*)+)$/; $1.split(',').map(&:to_i)
+        end
+        # We return [predicate string with index removed, expanded index]
+        [ key[0 ... key.size - subs.size], index ]
+      else
+        [ key, nil ]
+      end
+    end
+
+    def fetch_by_index(ary, index)
+      case index
+        when Array
+          ary.values_at(*index).compact
+        when Integer
+          v = ary.at(index)
+          v.nil? ? [] : [v]
+        when Range
+          ary[index]
+        else
+          fail ArgumentError, "weird index: #{index.inspect}"
+      end
+    end
+
+    def normal_key?(k)
+      [Integer, Range].any?{|cls| k.is_a?(cls) }
+    end
 
     # Returns all keys that match if method == :select, the first if
     # method == :find
     def all(key, method = :select)
-      return super if key.is_a?(Integer)
-      return nil if key.nil?
+      return [self[key]] if normal_key?(key)
+      return [] if key.nil?
 
+      key, index = extract_index(key)
       predicate = expand_predicate(key)
+      effective_method = index.nil? ? method : :select
 
-      send(method) do |v|
+      result = send(effective_method) do |v|
         predicate === v.named_array_identifier
       end
+
+      result = fetch_by_index(result, index) if index
+      Array(result).to_named_array
     end
+    indexable_wrapper :all
 
 
     # first matching item
     def [](key)
-      return super if key.is_a?(Integer)
-      all(key, :find)
+      return super if normal_key?(key)
+      all(key).first
     end
 
+    def first(key = nil)
+      return super() if key.nil?
+      all(key, :find).first
+    end
 
     # first matching item, or raises KeyError
     def fetch(key)
-      all(key, :find) or raise KeyError
+      first(key) or raise KeyError
     end
 
 
     def key?(key)
-      !! all(key, :find)
+      !! first(key)
     end
     alias_method :exist?, :key?
 
@@ -157,17 +238,24 @@ module Cult
     def with(**kw)
       fail ArgumentError, "with requires exactly one predicate" if kw.size != 1
 
-      method, predicate = kw.first
+      key, predicate = kw.first
       predicate = expand_predicate(predicate)
 
       select do |candidate|
-        method = ["names_for_#{method}", method].find do |m|
+        methods = [key, "query_for_#{key}", "names_for_#{key}"].select do |m|
           candidate.respond_to?(m)
         end
 
-        if method
-          result = Array(candidate.send(method))
-          result.any? {|r| predicate === r }
+        methods.any? do |method|
+          Array(candidate.send(method)).any? do |r|
+            begin
+              predicate === r
+            rescue
+              # We're going to assume this is a result of a string
+              # comparison to a custom #==
+              false
+            end
+          end
         end
       end
     end

data/lib/cult/node.rb

@@ -1,17 +1,40 @@
 require 'fileutils'
 require 'shellwords'
+require 'json'
 
 require 'cult/role'
 
 module Cult
   class Node < Role
+    class << self
+      attr_accessor :marshal_exclude
+    end
+
+    self.marshal_exclude = [:@project]
+
+    def marshal_dump
+      instance_variables.reject do |key|
+        self.class.marshal_exclude.include?(key)
+      end.map do |key|
+        [key, instance_variable_get(key)]
+      end.to_h
+    end
+
+    def marshal_load(vars)
+      vars.each do |key, value|
+        unless self.class.marshal_exclude.include?(key)
+          instance_variable_set(key, value)
+        end
+      end
+      self.project = Cult.project
+    end
+
     def self.from_data!(project, data)
       node = by_name(project, data[:name])
       raise Errno::EEXIST if node.exist?
 
       FileUtils.mkdir_p(node.path)
-      File.write(project.dump_name(node.node_path),
-                 project.dump_object(data))
+      File.write(node.node_path, JSON.pretty_generate(data))
 
       node.generate_ssh_keys!
 
@@ -19,10 +42,15 @@ module Cult
     end
 
     delegate_to_definition :host
+    delegate_to_definition :zone
+    delegate_to_definition :size
+    delegate_to_definition :image
+    delegate_to_definition :provider_name, :provider
     delegate_to_definition :ipv4_public
     delegate_to_definition :ipv4_private
     delegate_to_definition :ipv6_public
     delegate_to_definition :ipv6_private
+    delegate_to_definition :created_at
 
 
     def self.path(project)
@@ -31,17 +59,21 @@ module Cult
 
 
     def node_path
-      File.join(path, 'node')
+      File.join(path, 'node.json')
+    end
+
+    def exist?
+      File.exist?(state_path)
     end
 
 
     def state_path
-      File.join(path, 'state')
+      File.join(path, 'state.json')
     end
 
 
     def definition_path
-      [ node_path, state_path ]
+      [ extra_path, state_path, node_path ]
     end
 
 
@@ -50,8 +82,8 @@ module Cult
     end
 
 
-    def extra_file
-      File.join(path, 'extra')
+    def extra_path
+      File.join(path, 'extra.json')
     end
 
 
@@ -59,12 +91,9 @@ module Cult
       definition.direct('roles') || super
     end
 
-    def provider
-      project.providers[definition['provider']]
-    end
 
-    def names_for_provider
-      [ provider&.name ]
+    def provider
+      project.providers[provider_name]
     end
 
     alias_method :roles, :parent_roles
@@ -74,10 +103,21 @@ module Cult
       File.join(path, 'ssh.pub')
     end
 
+
     def ssh_private_key_file
       File.join(path, 'ssh.key')
     end
 
+    def ssh_known_hosts_file
+      File.join(path, 'ssh.known-host')
+    end
+
+    def ssh_port
+      # Moving SSH ports for security is lame.
+      definition['ssh_port'] || 22
+    end
+
+
     def generate_ssh_keys!
       esc = ->(s) { Shellwords.escape(s) }
       tmp_public = ssh_private_key_file + '.pub'
@@ -88,10 +128,97 @@ module Cult
             "-f #{esc.(ssh_private_key_file)} && " +
             "mv #{esc.(tmp_public)} #{esc.(ssh_public_key_file)}"
       %x(#{cmd})
+
       unless $?.success?
         fail "Couldn't generate SSH key, command: #{cmd}"
       end
+
+      File.chmod(0600, ssh_private_key_file)
+    end
+
+
+    def addr(access, protocol = project.default_ip_protocol)
+      fail ArgumentError unless [:public, :private].include?(access)
+      fail ArgumentError unless [:ipv4, :ipv6].include?(protocol)
+      send("#{protocol}_#{access}")
+    end
+
+
+    def same_network?(other)
+      [provider, zone] == [other.provider, other.zone]
+    end
+
+
+    def addr_from(other, protocol = project.default_ip_protocol)
+      same_network?(other) ? addr(:private, protocol) : addr(:public, protocol)
+    end
+
+
+    def cluster(*preds)
+      unless (preds - [:provider, :zone]).empty?
+        fail ArgumentError, "invalid predicate: #{preds.inspect}"
+      end
+
+      preds.push(:provider) if preds.include?(:zone)
+
+      col = project.nodes
+      preds.each do |pred|
+        col = col.select do |v|
+          v.send(pred) == send(pred)
+        end
+      end
+      col
+    end
+
+
+    def peers(*preds)
+      cluster(*preds).reject{|v| v == self}
+    end
+
+
+    def provider_peers
+      peers(:provider)
+    end
+
+
+    def leader(role, *preds)
+      c = cluster(*preds)
+      c = c.with(role: role) if role
+      c = c.sort_by(&:created_at)
+      c.first
+    end
+
+
+    def provider_leader(role = nil)
+      leader(role, :provider)
     end
 
+
+    def provider_leader?(role = nil)
+      provider_leader(role) == self
+    end
+
+
+    def zone_leader(role = nil)
+      leader(role, :zone)
+    end
+
+
+    def zone_leader?(role = nil)
+      zone_leader(role) == self
+    end
+
+
+    def zone_peers
+      peers(:zone)
+    end
+
+    def names_for_provider
+      [ provider_name ]
+    end
+
+    def names_for_zone
+      [zone]
+    end
   end
 end

data/lib/cult/paramap.rb

@@ -0,0 +1,209 @@
+# Cult.paramap runs a block in forked-off parallel processes.  There are very
+# little restrictions on what can be done in the block, but:
+#
+#  1. The value returned or any exceptions raised need to be Marshal-able.
+#  2. The blocks actually need to resume execution, so things like `exec`
+#     cause problems.
+#
+# The result of paramap is an array corresponding with each value ran through
+# the block.  There are a few exception strategies:
+#
+# 1. :raise The first job to throw an exception halts all further work and the
+#    exception is passed up
+# 2. :collect All jobs are allowed to complete, any exceptions encounted are
+#    tagged on the results in an `exception` method, which returns an array
+#    each element will either be 'nil' for no exception, or the Exception
+#    object the job raised.
+
+module Cult
+  class Paramap
+    class Job
+      attr_reader :ident, :value, :block
+      attr_reader :pid, :pipe
+
+      def initialize(ident, value, block)
+        @ident, @value, @block = ident, value, block
+
+        @pipe = IO.pipe
+        @pid = fork do
+          @pipe[0].close
+          prepare_forked_environment!
+          begin
+            write_response!('=', block.call(value))
+          rescue Exception => e
+            write_response!('!', e)
+          end
+        end
+        @pipe[1].close
+      end
+
+      def prepare_forked_environment!
+        # Stub out things that have caused a problem in the past.
+        Kernel.send(:define_method, :exec) do |*a|
+          fail "don't use Kernel\#exec inside of a paramap job"
+        end
+      end
+
+      def write_response!(scode, obj)
+        fail unless ['!', '='].include?(scode)
+        begin
+          pipe[1].write(scode + Marshal.dump(obj))
+        rescue TypeError => e
+          # Unmarshallable
+          raise unless e.message.match(/_dump_data/)
+          pipe[1].write(scode + Marshal.dump(nil))
+        end
+        pipe[1].flush
+        pipe[1].close
+      end
+
+      def fetch_response!
+        unless pipe[0].closed?
+          data = @pipe[0].read
+
+          scode = data[0]
+          fail unless ['!', '='].include?(scode)
+
+          data = data[1..-1]
+          ivar = (scode == '!') ? :exception : :result
+          begin
+            obj = Marshal.load(data)
+          rescue
+            obj = nil
+          end
+          instance_variable_set("@#{ivar}", obj)
+          pipe[0].close
+        end
+      end
+
+      def result
+        fetch_response!
+        @result
+      end
+
+      def exception
+        fetch_response!
+        @exception
+      end
+
+      def success?
+        exception.nil?
+      end
+    end
+
+    attr_reader :enum, :iter
+    attr_reader :block
+    attr_reader :job_queue
+    attr_reader :exception_strategy
+    attr_reader :exceptions
+    attr_reader :results
+    attr_reader :concurrent
+
+    def initialize(enum, concurrent: nil, exception_strategy:, &block)
+      @enum = enum
+      @iter = @enum.to_enum
+      @concurrent = concurrent || max_concurrent
+      @exception_strategy = exception_strategy
+      @block = block
+      @exceptions, @results = [], []
+      @job_queue = []
+    end
+
+    def max_concurrent
+      case (r = Cult.concurrency)
+        when :max
+          enum.respond_to?(:size) ? enum.size : 200
+        else
+          r
+      end
+    end
+
+    def handle_exception(job)
+      case exception_strategy
+        when :raise
+          raise job.exception
+        when :collect
+          exceptions.push(job.exception)
+        else
+          fail "Bad exception_strategy: #{exception_strategy}"
+      end
+    end
+
+    def handle_result(job)
+      results[job.ident] = job.result
+    end
+
+    def handle_response(job)
+      job.success? ? handle_result(job) : handle_exception(job)
+    end
+
+    def new_job_index
+      (@job_index ||= 0).tap do
+        @job_index += 1
+      end
+    end
+
+    def add_job(value)
+      job_queue.push(Job.new(new_job_index, value, block))
+    end
+
+    def job_by_pid(pid)
+      job_queue.find { |job| job.pid == pid }
+    end
+
+    def process_finished_job(job)
+      job_queue.delete(job)
+      handle_response(job)
+    end
+
+    def report_exceptions(results)
+      self_exceptions = self.exceptions
+      results.define_singleton_method(:exceptions) do
+        self_exceptions
+      end
+    end
+
+    def job_queue_full?
+      job_queue.size == concurrent
+    end
+
+    def more_tasks?
+      iter.peek
+      true
+    rescue StopIteration
+      false
+    end
+
+    def next_task
+      iter.next
+    end
+
+    def queue_next_task
+      add_job(next_task)
+    end
+
+    def wait_for_next_job_to_finish
+      if (job = job_by_pid(Process.waitpid))
+        process_finished_job(job)
+      end
+    end
+
+    def run
+      loop do
+        queue_next_task until job_queue_full? || !more_tasks?
+        break if job_queue.empty? && ! more_tasks?
+        wait_for_next_job_to_finish
+      end
+
+      report_exceptions(self.results)
+      self.results
+    end
+  end
+  private_constant :Paramap
+
+  module_function
+  def paramap(enum, concurrent: nil, exception: :raise, &block)
+    Paramap.new(enum, concurrent: concurrent,
+                exception_strategy: exception, &block).run
+  end
+end

data/lib/cult/project.rb

@@ -1,8 +1,6 @@
 require 'securerandom'
 require 'shellwords'
 require 'json'
-require 'yaml'
-
 
 module Cult
   class Project
@@ -10,8 +8,10 @@ module Cult
 
     attr_reader :path
     attr_accessor :cult_version
+    attr_accessor :default_ip_protocol
 
     def initialize(path)
+      @default_ip_protocol = :ipv4
       @path = path
     end
 
@@ -127,21 +127,6 @@ module Cult
     end
 
 
-    def dump_yaml?
-      !! (ENV['CULT_DUMP'] || '').match(/^yaml$/i)
-    end
-
-
-    def dump_object(obj)
-      dump_yaml? ? YAML.dump(obj) : JSON.pretty_generate(obj)
-    end
-
-
-    def dump_name(basename)
-      basename + (dump_yaml? ? '.yml' : '.json')
-    end
-
-
     def env
       ENV['CULT_ENV'] || begin
         if git_branch&.match(/\bdev(el(opment)?)?\b/)

data/lib/cult/provider.rb

@@ -43,7 +43,9 @@ module Cult
 
 
     def definition_path
-      [File.join(path, "provider"), File.join(path, "defaults")]
+      [ File.join(path, "extra.json"),
+        File.join(path, "defaults.json"),
+        File.join(path, "provider.json") ]
     end