ntl-orchestra 0.9.0

data/lib/orchestra/operation.rb

@@ -0,0 +1,49 @@
+module Orchestra
+  class Operation < Module
+    extend Forwardable
+
+    def_delegators :@default_run_list, :node_names, :provisions, :dependencies,
+      :optional_dependencies, :required_dependencies
+
+    attr :registry, :result, :nodes
+
+    def initialize args = {}
+      @result, @command, @nodes = Util.extract_key_args args,
+        :result, :command => false, :nodes => {}
+      @default_run_list = RunList.build nodes, result, []
+    end
+
+    def process output
+      output.select do |key, _| key = result end
+    end
+
+    def start_performance *args
+      conductor, input = extract_args args
+      run_list = RunList.build nodes, result, input.keys
+      performance = Performance.new conductor, run_list, input
+      yield performance if block_given?
+      performance.publish :operation_entered, name, input
+      performance
+    end
+
+    def perform *args, &block
+      performance = start_performance *args, &block
+      performance.perform
+      output = performance.extract_result result
+      performance.publish :operation_exited, name, output
+      @command ? nil : output
+    end
+
+    def command?
+      @command ? true : false
+    end
+
+    private
+
+    def extract_args args
+      conductor = args.size > 1 ? args.shift : Conductor.new
+      input = args.fetch 0 do {} end
+      [conductor, input]
+    end
+  end
+end

data/lib/orchestra/performance.rb

@@ -0,0 +1,137 @@
+module Orchestra
+  class Performance
+    include Observable
+    extend Forwardable
+
+    def_delegators :@run_list, :node_names, :provisions, :dependencies,
+      :optional_dependencies, :required_dependencies
+
+    attr :conductor, :input, :state, :registry, :run_list
+
+    def initialize conductor, run_list, input
+      @conductor = conductor
+      @input = input.dup
+      @run_list = run_list
+      @registry = conductor.build_registry self
+      @state = registry.merge input
+    end
+
+    def perform
+      ensure_inputs_are_present!
+      run_list.each do |name, node| process name, node end
+    rescue => error
+      publish :error_raised, error
+      raise error
+    end
+
+    def process name, node
+      input = input_for node
+      publish :node_entered, name, input
+      output = perform_node node
+      publish :node_exited, name, output
+      state.merge! output
+    end
+
+    def perform_node node
+      Movement.perform node, self
+    end
+
+    def ensure_inputs_are_present!
+      has_dep = state.method :[]
+      missing_input = required_dependencies.reject &has_dep
+      raise MissingInputError.new missing_input unless missing_input.empty?
+    end
+
+    def input_for node
+      state.reject do |key, val|
+        registry[key] == val or not node.dependencies.include? key
+      end
+    end
+
+    def extract_result result
+      state.fetch result
+    end
+
+    def publish event, *payload
+      changed
+      notify_observers event, *payload
+    end
+
+    def thread_pool
+      conductor.thread_pool
+    end
+
+    class Movement
+      def self.perform node, *args
+        if node.is_a? Operation
+          klass = EmbeddedOperation
+        else
+          klass = node.collection ? CollectionMovement : self
+        end
+        instance = klass.new node, *args
+        node.process instance.perform
+      end
+
+      attr :context, :node, :performance
+
+      def initialize node, performance
+        @node = node
+        @performance = performance
+        @context = build_context performance
+      end
+
+      def perform
+        context.perform
+      end
+
+      def build_context performance
+        node.build_context performance.state
+      end
+    end
+
+    class CollectionMovement < Movement
+      def perform
+        batch, output = prepare_collection
+        jobs = enqueue_jobs batch do |result, index| output[index] = result end
+        jobs.each &:wait
+        output
+      end
+
+      def enqueue_jobs batch, &block
+        batch.map.with_index do |element, index|
+          enqueue_job element, index, &block
+        end
+      end
+
+      def enqueue_job element, index
+        performance.thread_pool.enqueue do
+          result = context.perform element
+          yield [result, index]
+        end
+      end
+
+      def prepare_collection
+        batch = context.fetch_collection
+        output = [nil] * batch.size
+        [batch, output]
+      end
+    end
+
+    class EmbeddedOperation < Movement
+      def perform
+        super
+        context.state.select do |k,_| k == node.result end
+      end
+
+      def build_context performance
+        conductor = performance.registry[:conductor]
+        copy_observers = conductor.method :copy_observers
+        node.start_performance conductor, input, &copy_observers
+      end
+
+      def input
+        performance.state
+      end
+    end
+  end
+end

data/lib/orchestra/recording.rb

@@ -0,0 +1,83 @@
+module Orchestra
+  class Recording
+    attr :input, :output, :services
+
+    def initialize
+      @services = Hash.new do |hsh, service_name| hsh[service_name] = [] end
+    end
+
+    def update event_name, *args
+      case event_name
+      when :service_accessed then
+        service_name, recording = args
+        @services[service_name] << recording
+      when :operation_entered then
+        _, @input = args
+      when :operation_exited then
+        _, @output = args
+      else
+      end
+    end
+
+    def to_h
+      {
+        :input => input,
+        :output => output,
+        :service_recordings => services,
+      }
+    end
+
+    def self.replay operation, input, service_recordings
+      replayed_services = {}
+      service_recordings.each do |svc, service_recording|
+        replayed_services[svc] = Playback.build service_recording
+      end
+      conductor = Conductor.new replayed_services
+      conductor.perform operation, input
+    end
+
+    class Playback < BasicObject
+      attr :mocks
+
+      def initialize mocks
+        @mocks = mocks
+      end
+
+      def respond_to? meth
+        mocks.has_key? meth
+      end
+
+      def self.build service_recording
+        factory = Factory.new
+        factory.build service_recording
+      end
+
+      class Factory
+        attr :klass, :mocks
+
+        def initialize
+          @klass = Class.new Playback
+          @mocks = Hash.new do |hsh, meth| hsh[meth] = {} end
+        end
+
+        def build service_recording
+          record = method :<<
+          service_recording.each &record
+          klass.new mocks
+        end
+
+        def << record
+          method = record[:method].to_sym
+          unless klass.instance_methods.include? method
+            klass.send :define_method, method do |*args| mocks[method][args] end
+          end
+          mocks[method][record[:input]] = record[:output]
+        end
+
+        def singleton
+          singleton = class << instance ; self end
+        end
+      end
+    end
+  end
+end

data/lib/orchestra/run_list.rb

@@ -0,0 +1,171 @@
+module Orchestra
+  class RunList
+    def self.build nodes, result, input_names
+      builder = Builder.new result, input_names
+      builder.merge! nodes
+      builder.build
+    end
+
+    include Enumerable
+
+    def initialize nodes
+      @nodes = nodes
+      @nodes.freeze
+      freeze
+    end
+
+    def each &block
+      return to_enum :each unless block_given?
+      @nodes.each &block
+    end
+
+    def node_names
+      @nodes.keys
+    end
+
+    def dependencies
+      optional = collect_from_nodes :optional_dependencies
+      required = collect_from_nodes :required_dependencies
+      (optional + required).uniq
+    end
+
+    def optional_dependencies
+      collect_from_nodes :optional_dependencies
+    end
+
+    def provisions
+      collect_from_nodes :provisions
+    end
+
+    def required_dependencies
+      required_deps = collect_from_nodes :required_dependencies
+      required_deps - optional_dependencies - provisions
+    end
+
+    private
+
+    def collect_from_nodes method_name
+      set = @nodes.each_with_object Set.new do |(_, node), set|
+        deps = node.public_send method_name
+        deps.each &set.method(:<<)
+      end
+      set.to_a.tap &:sort!
+    end
+
+    class Builder
+      attr :input_names, :result
+
+      def initialize result, input_names = []
+        @input_names = input_names
+        @nodes_hash = {}
+        @required = [result]
+        @result = result
+        freeze
+      end
+
+      def merge! nodes
+        nodes.each do |name, node|
+          self[name] = node
+        end
+      end
+
+      def []= name, node
+        @nodes_hash[name] = node
+      end
+
+      def node_names
+        @nodes_hash.keys
+      end
+
+      def nodes
+        @nodes_hash.values
+      end
+
+      def build
+        sort!
+        prune!
+        RunList.new @nodes_hash
+      end
+
+      def sort!
+        sorter = Sorter.new @nodes_hash
+        sorter.sort!
+      end
+
+      def prune!
+        nodes.reverse_each.with_object [] do |node, removed|
+          removed.<< remove node and next unless required? node
+          require node
+        end
+      end
+
+      def remove node
+        @nodes_hash.reject! do |_, n| n == node end
+        node
+      end
+
+      def require node
+        supplied_by_input = input_names.method :include?
+        deps = node.required_dependencies.reject &supplied_by_input
+        @required.concat deps
+        true
+      end
+
+      def required? node
+        required = @required.method :include?
+        node.provisions.any? &required
+      end
+
+      class Sorter
+        include TSort
+
+        def initialize nodes_hash
+          @nodes = nodes_hash
+        end
+
+        def sort!
+          build_dependency_tree
+          tsort.each do |name|
+            @nodes[name] = @nodes.delete name
+          end
+        rescue TSort::Cyclic
+          raise CircularDependencyError.new
+        end
+
+        def build_dependency_tree
+          @hsh = @nodes.each_with_object Hash.new do |(name, node), hsh|
+          hsh[name] = build_dependencies_for node
+          end
+        end
+
+        def build_dependencies_for node
+          node.required_dependencies.each_with_object Set.new do |dep, set|
+            provider = provider_for dep
+            set << provider if provider
+          end
+        end
+
+        def tsort_each_node(&block)
+          @hsh.each_key &block
+        end
+
+        def tsort_each_child(name, &block)
+          deps = @hsh.fetch name
+          deps.each &block
+        end
+
+        def provider_for dep
+          @nodes.each do |name, node|
+            provisions = effective_provisions_for node, dep
+            return name if provisions.include? dep
+          end
+          nil
+        end
+
+        def effective_provisions_for node, dep
+          node.optional_dependencies | node.provisions
+        end
+      end
+    end
+  end
+end

data/lib/orchestra/thread_pool.rb

@@ -0,0 +1,163 @@
+module Orchestra
+  class ThreadPool
+    def self.build count
+      instance = new
+      instance.count = count
+      instance
+    end
+
+    def self.default
+      build 1
+    end
+
+    attr :queue, :timeout
+
+    def initialize args = {}
+      @timeout, _ = Util.extract_key_args args, :timeout_ms => 1000
+      @threads = Set.new
+      @dead_pool = Set.new
+      @pool_lock = Mutex.new
+      @queue = Queue.new
+      @jobs = {}
+    end
+
+    def enqueue &work
+      job = Job.new work
+      job.add_observer self
+      while_locked do queue << job end
+      job
+    end
+
+    def perform &work
+      job = enqueue &work
+      job.wait
+    end
+
+    def count
+      threads.size
+    end
+
+    def count= new_count
+      while_locked do
+        loop do
+          case @threads.size <=> new_count
+          when 0 then return
+          when -1 then add_thread!
+          when 1 then remove_thread!
+          end
+        end
+      end
+    end
+
+    def add_thread
+      while_locked do add_thread! end
+    end
+
+    def remove_thread
+      while_locked do remove_thread! end
+    end
+
+    def shutdown
+      self.count = 0
+    end
+
+    def status
+      while_locked do @threads.map &:status end
+    end
+
+    def threads
+      while_locked do @threads end
+    end
+
+    def update event, *;
+      reap_dead_pool if event == :failed
+    end
+
+    def with_timeout &block
+      Timeout.timeout Rational(timeout, 1000), &block
+    end
+
+    private
+
+    def add_thread!
+      wait_for_thread_count_to_change do
+        thr = Thread.new &method(:thread_loop)
+        @threads << thr
+      end
+      true
+    end
+
+    def reap_dead_pool
+      @dead_pool.each &:join
+      @dead_pool.clear
+    end
+
+    def remove_thread!
+      wait_for_thread_count_to_change do queue << :terminate end
+      sleep Rational(1, 10000) # FIXME
+      true
+    end
+
+    class Job
+      include Observable
+
+      attr :block, :error
+
+      def initialize block
+        @block = block
+        @output_queue = Queue.new
+      end
+
+      def done?
+        not @output_queue.empty?
+      end
+
+      def perform
+        @output_queue.push block.call
+      end
+
+      def set_error error
+        @error = error
+        @output_queue.push Failed
+      end
+
+      def wait
+        result = @output_queue.pop
+        changed
+        if result == Failed
+          notify_observers :failed, error
+          raise error
+        else
+          notify_observers :finished, result
+        end
+        result
+      end
+
+      Failed = Module.new
+    end
+
+    def thread_loop
+      Thread.current.abort_on_exception = false
+      until (job = queue.pop) == :terminate
+        job.perform
+      end
+    rescue => error
+      add_thread!
+      job.set_error error
+    ensure
+      @threads.delete Thread.current
+      @dead_pool << Thread.current
+    end
+
+    def wait_for_thread_count_to_change
+      old_count = queue.num_waiting
+      yield
+    ensure
+      Thread.pass while queue.num_waiting == old_count
+    end
+
+    def while_locked &block
+      @pool_lock.synchronize do with_timeout &block end
+    end
+  end
+end

data/lib/orchestra/util.rb

@@ -0,0 +1,98 @@
+module Orchestra
+  module Util
+    extend self
+
+    def extract_key_args hsh, *args
+      defaults, args = extract_hash args
+      unknown_args = hsh.keys - (args + defaults.keys)
+      missing_args = args - hsh.keys
+      unless unknown_args.empty? and missing_args.empty?
+        raise ArgumentError, key_arg_error(unknown_args, missing_args)
+      end
+      (args + defaults.keys).map do |arg|
+        hsh.fetch arg do defaults.fetch arg end
+      end
+    end
+
+    def extract_hash ary
+      if ary.last.is_a? Hash
+        hsh = ary.pop
+      else
+        hsh = {}
+      end
+      [hsh, ary]
+    end
+
+    def recursively_symbolize obj
+      case obj
+      when Array
+        obj.map &method(:recursively_symbolize)
+      when Hash then
+        obj.each_with_object Hash.new do |(k, v), out_hsh|
+          out_hsh[k.to_sym] = recursively_symbolize v
+        end
+      else obj
+      end
+    end
+
+    def to_lazy_thunk obj
+      if obj.respond_to? :to_proc and not obj.is_a? Symbol
+        obj
+      else
+        Proc.new do obj end
+      end
+    end
+
+    def to_camel_case str
+      str = "_#{str}"
+      str.gsub!(%r{_[a-z]}) { |snake| snake.slice(1).upcase }
+      str.gsub!('/', '::')
+      str
+    end
+
+    def to_snake_case str
+      str = str.gsub '::', '/'
+      # Convert FOOBar => FooBar
+      str.gsub! %r{[[:upper:]]{2,}} do |uppercase|
+        bit = uppercase[0]
+        bit << uppercase[1...-1].downcase
+        bit << uppercase[-1]
+        bit
+      end
+      # Convert FooBar => foo_bar
+      str.gsub! %r{[[:lower:]][[:upper:]]+[[:lower:]]} do |camel|
+        bit = camel[0]
+        bit << '_'
+        bit << camel[1..-1].downcase
+      end
+      str.downcase!
+      str
+    end
+
+    def demodulize str
+      split_namespaces(str).last
+    end
+
+    def deconstantize str
+      split_namespaces(str).first
+    end
+
+    private
+
+    def split_namespaces name
+      name.split '::'
+    end
+
+    def key_arg_error unknown, missing
+      str = "bad arguments. "
+      if unknown.any?
+        str.concat " unknown: #{unknown.join ', '}"
+        str.concat "; " if missing.any?
+      end
+      if missing.any?
+        str.concat " missing: #{missing.join ', '}"
+      end
+      str
+    end
+  end
+end

data/lib/orchestra/version.rb

@@ -0,0 +1,3 @@
+module Orchestra
+  VERSION = "0.9.0" unless defined? VERSION
+end