flatware 0.3.2 → 0.4.0

data/lib/flatware/rspec/formatters/console.rb

@@ -0,0 +1,33 @@
+module Flatware::RSpec::Formatters
+  class Console
+    attr_reader :formatter
+
+    def initialize(out, err)
+      ::RSpec::configuration.tty = true
+      ::RSpec::configuration.color = true
+      @formatter = ::RSpec::Core::Formatters::ProgressFormatter.new(out)
+    end
+
+    def progress(result)
+      formatter.send(message_for(result),nil)
+    end
+
+    def summarize(checkpoints)
+      result = checkpoints.reduce :+
+      if result
+        formatter.dump_failures result
+        formatter.dump_summary result.summary
+      end
+    end
+
+    private
+
+    def message_for(result)
+      {
+        passed:  :example_passed,
+        failed:  :example_failed,
+        pending: :example_pending
+      }.fetch result.progress
+    end
+  end
+end

data/lib/flatware/rspec/summary.rb

@@ -0,0 +1,40 @@
+require 'rspec/core/notifications'
+module Flatware
+  module RSpec
+    Summary = Struct.new(:duration, :examples, :failed_examples, :pending_examples, :load_time)
+
+    class Example
+      attr_reader :location_rerun_argument, :full_description
+      def initialize(rspec_example)
+        @full_description        = rspec_example.full_description
+        @location_rerun_argument = rspec_example.location_rerun_argument
+      end
+    end
+
+    class Summary
+      def +(other)
+        self.class.new duration + other.duration,
+          examples + other.examples,
+          failed_examples + other.failed_examples,
+          pending_examples + other.pending_examples,
+          load_time + other.load_time
+      end
+
+      def fully_formatted
+        ::RSpec::Core::Notifications::SummaryNotification.new(duration, examples, failed_examples, pending_examples, load_time).fully_formatted
+      end
+
+      def failure_count
+        failed_examples.size
+      end
+
+      def self.from_notification(summary)
+        serialized_examples = [summary.examples, summary.failed_examples, summary.pending_examples].map do |examples|
+          examples.map(&Example.method(:new))
+        end
+
+        new summary.duration, *serialized_examples, summary.load_time
+      end
+    end
+  end
+end

data/lib/flatware/serialized_exception.rb

@@ -1,20 +1,24 @@
 module Flatware
   class SerializedException
-    attr_reader :class, :message
+    attr_reader :class, :message, :cause
     attr_accessor :backtrace
-    def initialize(klass, message, backtrace)
-      @class, @message, @backtrace = serialized(klass), message, backtrace
+    def initialize(klass, message, backtrace, cause='')
+      @class, @message, @backtrace, @cause = serialized(klass), message, backtrace, cause
+    end
+
+    def self.from(exception)
+      new exception.class, exception.message, exception.backtrace, exception.cause
     end
 
     private
     def serialized(klass)
       SerializedClass.new(klass.to_s)
     end
+  end
 
-    class SerializedClass
-      attr_reader :name
-      alias to_s name
-      def initialize(name); @name = name end
-    end
+  class SerializedClass
+    attr_reader :name
+    alias to_s name
+    def initialize(name); @name = name end
   end
 end

data/lib/flatware/sink.rb

@@ -8,65 +8,70 @@ module Flatware
     end
 
     class Server
-      attr_reader :socket
-
-      def initialize(jobs, formatter, endpoint, options={})
-        @jobs, @formatter = jobs, formatter
-        options = {fail_fast: false}.merge options
-        @fail_fast = options[:fail_fast]
-        @socket = Flatware.socket(ZMQ::PULL, bind: endpoint)
+      attr_reader :sink, :dispatch, :poller, :workers, :checkpoints, :jobs, :formatter
+
+      def initialize(jobs:, formatter:, dispatch:, sink:, worker_count: 0)
+        @formatter = formatter
+        @jobs = group_jobs(jobs, worker_count)
+        @sink = Flatware.socket(ZMQ::PULL, bind: sink)
+        @dispatch = Flatware.socket(ZMQ::REP, bind: dispatch)
+        @poller = Poller.new(@sink, @dispatch)
+        @workers = Set.new(worker_count.times.to_a)
+        @checkpoints = []
       end
 
       def start
         trap 'INT' do
-          checkpoint_handler.summarize
+          puts "Interrupted!"
+          formatter.summarize checkpoints
           summarize_remaining
+          puts "\n\nCleaning up. Please wait...\n"
+          Flatware.close!
+          Process.waitall
+          puts "thanks."
           exit 1
         end
-
-        Flatware::Fireable::bind
         formatter.jobs jobs
-        listen
-      ensure
-        Flatware::Fireable::kill
-        Flatware.close
-      end
-
-      def checkpoint_handler
-        @checkpoint_handler ||= CheckpointHandler.new(formatter, fail_fast?)
+        listen.tap do
+          Flatware.close
+        end
       end
 
       def listen
-        until done?
+        que = jobs.dup
+        poller.each do |socket|
           message, content = socket.recv
+
           case message
+          when :ready
+            workers << content
+            job = que.shift
+            if job and not done?
+              dispatch.send job
+            else
+              workers.delete content
+              dispatch.send 'seppuku'
+            end
           when :checkpoint
-            checkpoint_handler.handle! content
+            checkpoints << content
           when :finished
             completed_jobs << content
             formatter.finished content
           else
             formatter.send message, content
           end
+          break if workers.empty? and done?
         end
-        checkpoint_handler.summarize
+        formatter.summarize(checkpoints)
         !failures?
-      rescue Error => e
-        raise unless e.message == "Interrupted system call"
       end
 
       private
 
       def failures?
-        checkpoint_handler.had_failures? || completed_jobs.any?(&:failed?)
-      end
-
-      def fail_fast?
-        @fail_fast
+        checkpoints.any?(&:failures?) || completed_jobs.any?(&:failed?)
       end
 
-      attr_reader :jobs, :formatter
-
       def summarize_remaining
         return if remaining_work.empty?
         formatter.summarize_remaining remaining_work
@@ -81,15 +86,20 @@ module Flatware
       end
 
       def done?
-        remaining_work.empty? || checkpoint_handler.done?
+        remaining_work.empty?
       end
 
       def remaining_work
         jobs - completed_jobs
       end
 
-      def fireable
-        @fireable ||= Fireable.new
+      def group_jobs(jobs, worker_count)
+        return jobs unless worker_count > 1
+        jobs.group_by.with_index do |_,i|
+          i % worker_count
+        end.values.map do |jobs|
+          Job.new(jobs.map(&:id), jobs.first.args)
+        end
       end
     end
   end

data/lib/flatware/socket.rb

@@ -1,4 +1,6 @@
 require 'ffi-rzmq'
+require 'securerandom'
+require 'logger'
 
 module Flatware
   Error = Class.new StandardError
@@ -14,20 +16,37 @@ module Flatware
 
   extend self
 
+  def logger
+    @logger ||= Logger.new($stderr)
+  end
+
+  def logger=(logger)
+    @logger = logger
+  end
+
   def socket(*args)
     context.socket(*args)
   end
 
-  def close
-    context.close
+  def close(force: false)
+    @ignore_errors = true if force
+    context.close(force: force)
     @context = nil
   end
 
+  def close!
+    close force: true
+  end
+
+  def socket_error
+    raise(Error, ZMQ::Util.error_string, caller) unless @ignore_errors
+  end
+
   def log(*message)
-    if verbose?
-      $stderr.print "#$0 "
-      $stderr.puts *message
-      $stderr.flush
+    if Exception === message.first
+      logger.error message.first
+    elsif verbose?
+      logger.info ([$0] + message).join(' ')
     end
     message
   end
@@ -38,7 +57,10 @@ module Flatware
   end
 
   def context
-    @context ||= Context.new
+    @context ||= begin
+                   @ignore_errors = nil
+                   Context.new
+                 end
   end
 
   class Context
@@ -49,8 +71,8 @@ module Flatware
       @sockets = []
     end
 
-    def socket(type, options={})
-      Socket.new(c.socket(type)).tap do |socket|
+    def socket(zmq_type, options={})
+      Socket.new(c.socket(zmq_type)).tap do |socket|
         sockets.push socket
         if port = options[:connect]
           socket.connect port
@@ -61,26 +83,34 @@ module Flatware
       end
     end
 
-    def close
-      sockets.each &:close
-      raise(Error, ZMQ::Util.error_string, caller) unless c.terminate == 0
+    def close(force: false)
+      sockets.each do |socket|
+        socket.setsockopt ZMQ::LINGER, 0
+      end if force
+      sockets.each(&:close)
+      Flatware::socket_error unless c.terminate == 0
       Flatware.log "terminated context"
     end
   end
 
   class Socket
-    attr_reader :s
+    attr_reader :socket
+
     def initialize(socket)
-      @s = socket
+      @socket = socket
     end
 
     def setsockopt(*args)
-      s.setsockopt(*args)
+      socket.setsockopt(*args)
+    end
+
+    def name
+      socket.name
     end
 
     def send(message)
-      result = s.send_string(Marshal.dump(message))
-      raise Error, ZMQ::Util.error_string, caller if result == -1
+      result = socket.send_string(Marshal.dump(message))
+      Flatware::socket_error if result == -1
       Flatware.log "#@type #@port send #{message}"
       message
     end
@@ -88,28 +118,62 @@ module Flatware
     def connect(port)
       @type = 'connected'
       @port = port
-      raise(Error, ZMQ::Util.error_string, caller) unless s.connect(port) == 0
+      Flatware::socket_error unless socket.connect(port) == 0
       Flatware.log "connect #@port"
     end
 
+    def monitor
+      name = "inproc://monitor#{SecureRandom.hex(10)}"
+      LibZMQ.zmq_socket_monitor(socket.socket, name, ZMQ::EVENT_ALL)
+      Monitor.new(name)
+    end
+
+    class Monitor
+      def initialize(port)
+        @socket = Flatware.socket ZMQ::PAIR
+        @socket.connect port
+      end
+
+      def recv
+        bytes = @socket.recv marshal: false
+        data = LibZMQ::EventData.new FFI::MemoryPointer.from_string bytes
+        event[data.event]
+      end
+
+      private
+
+      def event
+        ZMQ.constants.select do |c|
+          c.to_s =~ /^EVENT/
+        end.map do |s|
+          {s => ZMQ.const_get(s)}
+        end.reduce(:merge).invert
+      end
+    end
+
     def bind(port)
       @type = 'bound'
       @port = port
-      raise(Error, ZMQ::Util.error_string, caller) unless s.bind(port) == 0
+      Flatware::socket_error unless socket.bind(port) == 0
       Flatware.log "bind #@port"
     end
 
     def close
-      setsockopt ZMQ::LINGER, 0
-      raise(Error, ZMQ::Util.error_string, caller) unless s.close == 0
+      Flatware::socket_error unless socket.close == 0
       Flatware.log "close #@type #@port"
     end
 
-    def recv
+    def recv(block: true, marshal: true)
       message = ''
-      result = s.recv_string(message)
-      raise Error, ZMQ::Util.error_string, caller if result == -1
-      message = Marshal.load message
+      if block
+       result = socket.recv_string(message)
+       Flatware::socket_error if result == -1
+      else
+        socket.recv_string(message, ZMQ::NOBLOCK)
+      end
+      if message != '' and marshal
+        message = Marshal.load(message)
+      end
       Flatware.log "#@type #@port recv #{message}"
       message
     end

data/lib/flatware/version.rb

@@ -1,3 +1,3 @@
 module Flatware
-  VERSION = '0.3.2'
+  VERSION = '0.4.0'
 end

data/lib/flatware/worker.rb

@@ -6,37 +6,44 @@ module Flatware
     def initialize(id, runner, dispatch_endpoint, sink_endpoint)
       @id       = id
       @runner   = runner
-      @fireable = Fireable.new
       @sink     = Sink::Client.new sink_endpoint
       @task     = Flatware.socket ZMQ::REQ, connect: dispatch_endpoint
     end
 
-    def self.spawn(worker_count, runner, dispatch_endpoint, sink_endpoint)
-      worker_count.times do |i|
+    def self.spawn(count:, runner:, dispatch:, sink:)
+      count.times do |i|
         fork do
           $0 = "flatware worker #{i}"
           ENV['TEST_ENV_NUMBER'] = i.to_s
-          new(i, runner, dispatch_endpoint, sink_endpoint).listen
+          new(i, runner, dispatch, sink).listen
         end
       end
     end
 
     def listen
+      trap 'INT' do
+        Flatware.close!
+        @want_to_quit = true
+        exit(1)
+      end
+
       Sink.client = sink
       report_for_duty
-      fireable.until_fired task do |job|
+      loop do
+        job = task.recv
+        break if job == 'seppuku' or @want_to_quit
         job.worker = id
         sink.started job
         begin
           runner.run job.id, job.args
-        rescue Errno::ENOENT
+        rescue => e
+          Flatware.log e
           job.failed = true
-          sink.finished job
-          raise
         end
         sink.finished job
         report_for_duty
       end
+      Flatware.close unless @want_to_quit
     end
 
     private
@@ -44,7 +51,7 @@ module Flatware
     attr_reader :fireable, :task, :sink, :runner
 
     def report_for_duty
-      task.send 'ready'
+      task.send [:ready, id]
     end
   end
 end