flatware 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +83 -39
  3. data/lib/flatware-cucumber.rb +1 -0
  4. data/lib/flatware-rspec.rb +1 -0
  5. data/lib/flatware.rb +2 -10
  6. data/lib/flatware/broadcaster.rb +15 -0
  7. data/lib/flatware/cli.rb +10 -34
  8. data/lib/flatware/cucumber.rb +37 -12
  9. data/lib/flatware/cucumber/checkpoint.rb +28 -0
  10. data/lib/flatware/cucumber/cli.rb +22 -0
  11. data/lib/flatware/cucumber/formatter.rb +36 -84
  12. data/lib/flatware/{formatters/cucumber → cucumber/formatters}/console.rb +5 -3
  13. data/lib/flatware/{formatters/cucumber → cucumber/formatters}/console/summary.rb +3 -3
  14. data/lib/flatware/cucumber/result.rb +27 -0
  15. data/lib/flatware/cucumber/scenario_result.rb +38 -0
  16. data/lib/flatware/cucumber/step_result.rb +30 -0
  17. data/lib/flatware/poller.rb +2 -2
  18. data/lib/flatware/rspec.rb +28 -0
  19. data/lib/flatware/rspec/checkpoint.rb +29 -0
  20. data/lib/flatware/rspec/cli.rb +16 -0
  21. data/lib/flatware/rspec/example_notification.rb +21 -0
  22. data/lib/flatware/rspec/examples_notification.rb +24 -0
  23. data/lib/flatware/rspec/formatter.rb +50 -0
  24. data/lib/flatware/rspec/formatters/console.rb +33 -0
  25. data/lib/flatware/rspec/summary.rb +40 -0
  26. data/lib/flatware/serialized_exception.rb +12 -8
  27. data/lib/flatware/sink.rb +43 -33
  28. data/lib/flatware/socket.rb +89 -25
  29. data/lib/flatware/version.rb +1 -1
  30. data/lib/flatware/worker.rb +16 -9
  31. metadata +27 -55
  32. data/lib/flatware/checkpoint.rb +0 -28
  33. data/lib/flatware/checkpoint_handler.rb +0 -45
  34. data/lib/flatware/dispatcher.rb +0 -31
  35. data/lib/flatware/fireable.rb +0 -34
  36. data/lib/flatware/formatters.rb +0 -27
  37. data/lib/flatware/formatters/cucumber/http.rb +0 -83
  38. data/lib/flatware/result.rb +0 -25
  39. data/lib/flatware/scenario_decorator.rb +0 -22
  40. data/lib/flatware/scenario_result.rb +0 -36
  41. data/lib/flatware/step_result.rb +0 -27
@@ -0,0 +1,33 @@
1
+ module Flatware::RSpec::Formatters
2
+ class Console
3
+ attr_reader :formatter
4
+
5
+ def initialize(out, err)
6
+ ::RSpec::configuration.tty = true
7
+ ::RSpec::configuration.color = true
8
+ @formatter = ::RSpec::Core::Formatters::ProgressFormatter.new(out)
9
+ end
10
+
11
+ def progress(result)
12
+ formatter.send(message_for(result),nil)
13
+ end
14
+
15
+ def summarize(checkpoints)
16
+ result = checkpoints.reduce :+
17
+ if result
18
+ formatter.dump_failures result
19
+ formatter.dump_summary result.summary
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ def message_for(result)
26
+ {
27
+ passed: :example_passed,
28
+ failed: :example_failed,
29
+ pending: :example_pending
30
+ }.fetch result.progress
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,40 @@
1
+ require 'rspec/core/notifications'
2
+ module Flatware
3
+ module RSpec
4
+ Summary = Struct.new(:duration, :examples, :failed_examples, :pending_examples, :load_time)
5
+
6
+ class Example
7
+ attr_reader :location_rerun_argument, :full_description
8
+ def initialize(rspec_example)
9
+ @full_description = rspec_example.full_description
10
+ @location_rerun_argument = rspec_example.location_rerun_argument
11
+ end
12
+ end
13
+
14
+ class Summary
15
+ def +(other)
16
+ self.class.new duration + other.duration,
17
+ examples + other.examples,
18
+ failed_examples + other.failed_examples,
19
+ pending_examples + other.pending_examples,
20
+ load_time + other.load_time
21
+ end
22
+
23
+ def fully_formatted
24
+ ::RSpec::Core::Notifications::SummaryNotification.new(duration, examples, failed_examples, pending_examples, load_time).fully_formatted
25
+ end
26
+
27
+ def failure_count
28
+ failed_examples.size
29
+ end
30
+
31
+ def self.from_notification(summary)
32
+ serialized_examples = [summary.examples, summary.failed_examples, summary.pending_examples].map do |examples|
33
+ examples.map(&Example.method(:new))
34
+ end
35
+
36
+ new summary.duration, *serialized_examples, summary.load_time
37
+ end
38
+ end
39
+ end
40
+ end
@@ -1,20 +1,24 @@
1
1
  module Flatware
2
2
  class SerializedException
3
- attr_reader :class, :message
3
+ attr_reader :class, :message, :cause
4
4
  attr_accessor :backtrace
5
- def initialize(klass, message, backtrace)
6
- @class, @message, @backtrace = serialized(klass), message, backtrace
5
+ def initialize(klass, message, backtrace, cause='')
6
+ @class, @message, @backtrace, @cause = serialized(klass), message, backtrace, cause
7
+ end
8
+
9
+ def self.from(exception)
10
+ new exception.class, exception.message, exception.backtrace, exception.cause
7
11
  end
8
12
 
9
13
  private
10
14
  def serialized(klass)
11
15
  SerializedClass.new(klass.to_s)
12
16
  end
17
+ end
13
18
 
14
- class SerializedClass
15
- attr_reader :name
16
- alias to_s name
17
- def initialize(name); @name = name end
18
- end
19
+ class SerializedClass
20
+ attr_reader :name
21
+ alias to_s name
22
+ def initialize(name); @name = name end
19
23
  end
20
24
  end
@@ -8,65 +8,70 @@ module Flatware
8
8
  end
9
9
 
10
10
  class Server
11
- attr_reader :socket
12
-
13
- def initialize(jobs, formatter, endpoint, options={})
14
- @jobs, @formatter = jobs, formatter
15
- options = {fail_fast: false}.merge options
16
- @fail_fast = options[:fail_fast]
17
- @socket = Flatware.socket(ZMQ::PULL, bind: endpoint)
11
+ attr_reader :sink, :dispatch, :poller, :workers, :checkpoints, :jobs, :formatter
12
+
13
+ def initialize(jobs:, formatter:, dispatch:, sink:, worker_count: 0)
14
+ @formatter = formatter
15
+ @jobs = group_jobs(jobs, worker_count)
16
+ @sink = Flatware.socket(ZMQ::PULL, bind: sink)
17
+ @dispatch = Flatware.socket(ZMQ::REP, bind: dispatch)
18
+ @poller = Poller.new(@sink, @dispatch)
19
+ @workers = Set.new(worker_count.times.to_a)
20
+ @checkpoints = []
18
21
  end
19
22
 
20
23
  def start
21
24
  trap 'INT' do
22
- checkpoint_handler.summarize
25
+ puts "Interrupted!"
26
+ formatter.summarize checkpoints
23
27
  summarize_remaining
28
+ puts "\n\nCleaning up. Please wait...\n"
29
+ Flatware.close!
30
+ Process.waitall
31
+ puts "thanks."
24
32
  exit 1
25
33
  end
26
-
27
- Flatware::Fireable::bind
28
34
  formatter.jobs jobs
29
- listen
30
- ensure
31
- Flatware::Fireable::kill
32
- Flatware.close
33
- end
34
-
35
- def checkpoint_handler
36
- @checkpoint_handler ||= CheckpointHandler.new(formatter, fail_fast?)
35
+ listen.tap do
36
+ Flatware.close
37
+ end
37
38
  end
38
39
 
39
40
  def listen
40
- until done?
41
+ que = jobs.dup
42
+ poller.each do |socket|
41
43
  message, content = socket.recv
44
+
42
45
  case message
46
+ when :ready
47
+ workers << content
48
+ job = que.shift
49
+ if job and not done?
50
+ dispatch.send job
51
+ else
52
+ workers.delete content
53
+ dispatch.send 'seppuku'
54
+ end
43
55
  when :checkpoint
44
- checkpoint_handler.handle! content
56
+ checkpoints << content
45
57
  when :finished
46
58
  completed_jobs << content
47
59
  formatter.finished content
48
60
  else
49
61
  formatter.send message, content
50
62
  end
63
+ break if workers.empty? and done?
51
64
  end
52
- checkpoint_handler.summarize
65
+ formatter.summarize(checkpoints)
53
66
  !failures?
54
- rescue Error => e
55
- raise unless e.message == "Interrupted system call"
56
67
  end
57
68
 
58
69
  private
59
70
 
60
71
  def failures?
61
- checkpoint_handler.had_failures? || completed_jobs.any?(&:failed?)
62
- end
63
-
64
- def fail_fast?
65
- @fail_fast
72
+ checkpoints.any?(&:failures?) || completed_jobs.any?(&:failed?)
66
73
  end
67
74
 
68
- attr_reader :jobs, :formatter
69
-
70
75
  def summarize_remaining
71
76
  return if remaining_work.empty?
72
77
  formatter.summarize_remaining remaining_work
@@ -81,15 +86,20 @@ module Flatware
81
86
  end
82
87
 
83
88
  def done?
84
- remaining_work.empty? || checkpoint_handler.done?
89
+ remaining_work.empty?
85
90
  end
86
91
 
87
92
  def remaining_work
88
93
  jobs - completed_jobs
89
94
  end
90
95
 
91
- def fireable
92
- @fireable ||= Fireable.new
96
+ def group_jobs(jobs, worker_count)
97
+ return jobs unless worker_count > 1
98
+ jobs.group_by.with_index do |_,i|
99
+ i % worker_count
100
+ end.values.map do |jobs|
101
+ Job.new(jobs.map(&:id), jobs.first.args)
102
+ end
93
103
  end
94
104
  end
95
105
  end
@@ -1,4 +1,6 @@
1
1
  require 'ffi-rzmq'
2
+ require 'securerandom'
3
+ require 'logger'
2
4
 
3
5
  module Flatware
4
6
  Error = Class.new StandardError
@@ -14,20 +16,37 @@ module Flatware
14
16
 
15
17
  extend self
16
18
 
19
+ def logger
20
+ @logger ||= Logger.new($stderr)
21
+ end
22
+
23
+ def logger=(logger)
24
+ @logger = logger
25
+ end
26
+
17
27
  def socket(*args)
18
28
  context.socket(*args)
19
29
  end
20
30
 
21
- def close
22
- context.close
31
+ def close(force: false)
32
+ @ignore_errors = true if force
33
+ context.close(force: force)
23
34
  @context = nil
24
35
  end
25
36
 
37
+ def close!
38
+ close force: true
39
+ end
40
+
41
+ def socket_error
42
+ raise(Error, ZMQ::Util.error_string, caller) unless @ignore_errors
43
+ end
44
+
26
45
  def log(*message)
27
- if verbose?
28
- $stderr.print "#$0 "
29
- $stderr.puts *message
30
- $stderr.flush
46
+ if Exception === message.first
47
+ logger.error message.first
48
+ elsif verbose?
49
+ logger.info ([$0] + message).join(' ')
31
50
  end
32
51
  message
33
52
  end
@@ -38,7 +57,10 @@ module Flatware
38
57
  end
39
58
 
40
59
  def context
41
- @context ||= Context.new
60
+ @context ||= begin
61
+ @ignore_errors = nil
62
+ Context.new
63
+ end
42
64
  end
43
65
 
44
66
  class Context
@@ -49,8 +71,8 @@ module Flatware
49
71
  @sockets = []
50
72
  end
51
73
 
52
- def socket(type, options={})
53
- Socket.new(c.socket(type)).tap do |socket|
74
+ def socket(zmq_type, options={})
75
+ Socket.new(c.socket(zmq_type)).tap do |socket|
54
76
  sockets.push socket
55
77
  if port = options[:connect]
56
78
  socket.connect port
@@ -61,26 +83,34 @@ module Flatware
61
83
  end
62
84
  end
63
85
 
64
- def close
65
- sockets.each &:close
66
- raise(Error, ZMQ::Util.error_string, caller) unless c.terminate == 0
86
+ def close(force: false)
87
+ sockets.each do |socket|
88
+ socket.setsockopt ZMQ::LINGER, 0
89
+ end if force
90
+ sockets.each(&:close)
91
+ Flatware::socket_error unless c.terminate == 0
67
92
  Flatware.log "terminated context"
68
93
  end
69
94
  end
70
95
 
71
96
  class Socket
72
- attr_reader :s
97
+ attr_reader :socket
98
+
73
99
  def initialize(socket)
74
- @s = socket
100
+ @socket = socket
75
101
  end
76
102
 
77
103
  def setsockopt(*args)
78
- s.setsockopt(*args)
104
+ socket.setsockopt(*args)
105
+ end
106
+
107
+ def name
108
+ socket.name
79
109
  end
80
110
 
81
111
  def send(message)
82
- result = s.send_string(Marshal.dump(message))
83
- raise Error, ZMQ::Util.error_string, caller if result == -1
112
+ result = socket.send_string(Marshal.dump(message))
113
+ Flatware::socket_error if result == -1
84
114
  Flatware.log "#@type #@port send #{message}"
85
115
  message
86
116
  end
@@ -88,28 +118,62 @@ module Flatware
88
118
  def connect(port)
89
119
  @type = 'connected'
90
120
  @port = port
91
- raise(Error, ZMQ::Util.error_string, caller) unless s.connect(port) == 0
121
+ Flatware::socket_error unless socket.connect(port) == 0
92
122
  Flatware.log "connect #@port"
93
123
  end
94
124
 
125
+ def monitor
126
+ name = "inproc://monitor#{SecureRandom.hex(10)}"
127
+ LibZMQ.zmq_socket_monitor(socket.socket, name, ZMQ::EVENT_ALL)
128
+ Monitor.new(name)
129
+ end
130
+
131
+ class Monitor
132
+ def initialize(port)
133
+ @socket = Flatware.socket ZMQ::PAIR
134
+ @socket.connect port
135
+ end
136
+
137
+ def recv
138
+ bytes = @socket.recv marshal: false
139
+ data = LibZMQ::EventData.new FFI::MemoryPointer.from_string bytes
140
+ event[data.event]
141
+ end
142
+
143
+ private
144
+
145
+ def event
146
+ ZMQ.constants.select do |c|
147
+ c.to_s =~ /^EVENT/
148
+ end.map do |s|
149
+ {s => ZMQ.const_get(s)}
150
+ end.reduce(:merge).invert
151
+ end
152
+ end
153
+
95
154
  def bind(port)
96
155
  @type = 'bound'
97
156
  @port = port
98
- raise(Error, ZMQ::Util.error_string, caller) unless s.bind(port) == 0
157
+ Flatware::socket_error unless socket.bind(port) == 0
99
158
  Flatware.log "bind #@port"
100
159
  end
101
160
 
102
161
  def close
103
- setsockopt ZMQ::LINGER, 0
104
- raise(Error, ZMQ::Util.error_string, caller) unless s.close == 0
162
+ Flatware::socket_error unless socket.close == 0
105
163
  Flatware.log "close #@type #@port"
106
164
  end
107
165
 
108
- def recv
166
+ def recv(block: true, marshal: true)
109
167
  message = ''
110
- result = s.recv_string(message)
111
- raise Error, ZMQ::Util.error_string, caller if result == -1
112
- message = Marshal.load message
168
+ if block
169
+ result = socket.recv_string(message)
170
+ Flatware::socket_error if result == -1
171
+ else
172
+ socket.recv_string(message, ZMQ::NOBLOCK)
173
+ end
174
+ if message != '' and marshal
175
+ message = Marshal.load(message)
176
+ end
113
177
  Flatware.log "#@type #@port recv #{message}"
114
178
  message
115
179
  end
@@ -1,3 +1,3 @@
1
1
  module Flatware
2
- VERSION = '0.3.2'
2
+ VERSION = '0.4.0'
3
3
  end
@@ -6,37 +6,44 @@ module Flatware
6
6
  def initialize(id, runner, dispatch_endpoint, sink_endpoint)
7
7
  @id = id
8
8
  @runner = runner
9
- @fireable = Fireable.new
10
9
  @sink = Sink::Client.new sink_endpoint
11
10
  @task = Flatware.socket ZMQ::REQ, connect: dispatch_endpoint
12
11
  end
13
12
 
14
- def self.spawn(worker_count, runner, dispatch_endpoint, sink_endpoint)
15
- worker_count.times do |i|
13
+ def self.spawn(count:, runner:, dispatch:, sink:)
14
+ count.times do |i|
16
15
  fork do
17
16
  $0 = "flatware worker #{i}"
18
17
  ENV['TEST_ENV_NUMBER'] = i.to_s
19
- new(i, runner, dispatch_endpoint, sink_endpoint).listen
18
+ new(i, runner, dispatch, sink).listen
20
19
  end
21
20
  end
22
21
  end
23
22
 
24
23
  def listen
24
+ trap 'INT' do
25
+ Flatware.close!
26
+ @want_to_quit = true
27
+ exit(1)
28
+ end
29
+
25
30
  Sink.client = sink
26
31
  report_for_duty
27
- fireable.until_fired task do |job|
32
+ loop do
33
+ job = task.recv
34
+ break if job == 'seppuku' or @want_to_quit
28
35
  job.worker = id
29
36
  sink.started job
30
37
  begin
31
38
  runner.run job.id, job.args
32
- rescue Errno::ENOENT
39
+ rescue => e
40
+ Flatware.log e
33
41
  job.failed = true
34
- sink.finished job
35
- raise
36
42
  end
37
43
  sink.finished job
38
44
  report_for_duty
39
45
  end
46
+ Flatware.close unless @want_to_quit
40
47
  end
41
48
 
42
49
  private
@@ -44,7 +51,7 @@ module Flatware
44
51
  attr_reader :fireable, :task, :sink, :runner
45
52
 
46
53
  def report_for_duty
47
- task.send 'ready'
54
+ task.send [:ready, id]
48
55
  end
49
56
  end
50
57
  end