distribot 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +13 -0
  3. data/.rspec +3 -0
  4. data/.rubocop.yml +3 -0
  5. data/.travis.yml +10 -0
  6. data/Dockerfile +9 -0
  7. data/Gemfile +6 -0
  8. data/Gemfile.lock +153 -0
  9. data/LICENSE +201 -0
  10. data/README.md +107 -0
  11. data/Rakefile +16 -0
  12. data/bin/distribot.flow-created +6 -0
  13. data/bin/distribot.flow-finished +6 -0
  14. data/bin/distribot.handler-finished +6 -0
  15. data/bin/distribot.phase-finished +6 -0
  16. data/bin/distribot.phase-started +6 -0
  17. data/bin/distribot.task-finished +6 -0
  18. data/distribot.gemspec +35 -0
  19. data/docker-compose.yml +29 -0
  20. data/examples/controller +168 -0
  21. data/examples/distribot.eye +49 -0
  22. data/examples/status +38 -0
  23. data/examples/worker +135 -0
  24. data/lib/distribot/connector.rb +162 -0
  25. data/lib/distribot/flow.rb +200 -0
  26. data/lib/distribot/flow_created_handler.rb +12 -0
  27. data/lib/distribot/flow_finished_handler.rb +12 -0
  28. data/lib/distribot/handler.rb +40 -0
  29. data/lib/distribot/handler_finished_handler.rb +29 -0
  30. data/lib/distribot/phase.rb +46 -0
  31. data/lib/distribot/phase_finished_handler.rb +19 -0
  32. data/lib/distribot/phase_handler.rb +15 -0
  33. data/lib/distribot/phase_started_handler.rb +69 -0
  34. data/lib/distribot/task_finished_handler.rb +37 -0
  35. data/lib/distribot/worker.rb +148 -0
  36. data/lib/distribot.rb +108 -0
  37. data/provision/nodes.sh +80 -0
  38. data/provision/templates/fluentd.conf +27 -0
  39. data/spec/distribot/bunny_connector_spec.rb +196 -0
  40. data/spec/distribot/connection_sharer_spec.rb +34 -0
  41. data/spec/distribot/connector_spec.rb +63 -0
  42. data/spec/distribot/flow_created_handler_spec.rb +32 -0
  43. data/spec/distribot/flow_finished_handler_spec.rb +32 -0
  44. data/spec/distribot/flow_spec.rb +661 -0
  45. data/spec/distribot/handler_finished_handler_spec.rb +112 -0
  46. data/spec/distribot/handler_spec.rb +32 -0
  47. data/spec/distribot/module_spec.rb +163 -0
  48. data/spec/distribot/multi_subscription_spec.rb +37 -0
  49. data/spec/distribot/phase_finished_handler_spec.rb +61 -0
  50. data/spec/distribot/phase_started_handler_spec.rb +150 -0
  51. data/spec/distribot/subscription_spec.rb +40 -0
  52. data/spec/distribot/task_finished_handler_spec.rb +71 -0
  53. data/spec/distribot/worker_spec.rb +281 -0
  54. data/spec/fixtures/simple_flow.json +49 -0
  55. data/spec/spec_helper.rb +74 -0
  56. metadata +371 -0
data/lib/distribot.rb ADDED
@@ -0,0 +1,108 @@
1
+
2
+ require 'active_support/core_ext/object'
3
+ require 'active_support/json'
4
+ require 'redis'
5
+ require 'distribot/flow'
6
+ require 'distribot/phase'
7
+ require 'distribot/phase_handler'
8
+ require 'distribot/handler'
9
+ require 'distribot/flow_created_handler'
10
+ require 'distribot/phase_started_handler'
11
+ require 'distribot/worker'
12
+ require 'distribot/task_finished_handler'
13
+ require 'distribot/handler_finished_handler'
14
+ require 'distribot/phase_finished_handler'
15
+ require 'distribot/flow_finished_handler'
16
+ require 'distribot/connector'
17
+ require 'syslog/logger'
18
+ require 'logstash-logger'
19
+ require 'concurrent'
20
+
21
+ module Distribot
22
+ class << self
23
+ attr_accessor :config, :did_configure, :connector, :redis, :debug, :logger
24
+ @config = OpenStruct.new
25
+ @did_configure = false
26
+ @connector = nil
27
+
28
+ def reset_configuration!
29
+ self.config = OpenStruct.new
30
+ self.did_configure = false
31
+ self.redis = nil
32
+ end
33
+
34
+ def configure(&block)
35
+ reset_configuration!
36
+ @did_configure = true
37
+ block.call(configuration)
38
+ # Now set defaults for things that aren't defined:
39
+ configuration.redis_prefix ||= 'distribot'
40
+ configuration.queue_prefix ||= 'distribot'
41
+ end
42
+
43
+ def connector
44
+ @connector ||= BunnyConnector.new(configuration.rabbitmq_url)
45
+ end
46
+
47
+ def configuration
48
+ unless @did_configure
49
+ reset_configuration!
50
+ configure do |config|
51
+ config.redis_url = ENV['DISTRIBOT_REDIS_URL']
52
+ config.rabbitmq_url = ENV['DISTRIBOT_RABBITMQ_URL']
53
+ end
54
+ end
55
+ self.config
56
+ end
57
+
58
+ def queue_exists?(name)
59
+ connector.queue_exists?(name)
60
+ end
61
+
62
+ def redis
63
+ # Redis complains if we pass it a nil url. Better to not pass a url:
64
+ @redis ||= if configuration.redis_url
65
+ Redis.new(url: configuration.redis_url)
66
+ else
67
+ Redis.new
68
+ end
69
+ end
70
+
71
+ def debug=(value)
72
+ @debug = value ? true : false
73
+ end
74
+
75
+ def debug
76
+ @debug ||= false
77
+ end
78
+
79
+ def redis_id(type, id)
80
+ "#{configuration.redis_prefix}-#{type}:#{id}"
81
+ end
82
+
83
+ def publish!(topic, data)
84
+ connector.publish(topic, data)
85
+ end
86
+
87
+ def subscribe(topic, options = {}, &block)
88
+ connector.subscribe(topic, options) do |message|
89
+ block.call(message)
90
+ end
91
+ end
92
+
93
+ def broadcast!(topic, data)
94
+ connector.broadcast(topic, data)
95
+ end
96
+
97
+ def subscribe_multi(topic, options = {}, &block)
98
+ connector.subscribe_multi(topic, options) do |message|
99
+ block.call(message)
100
+ end
101
+ end
102
+
103
+ def logger
104
+ @logger ||= LogStashLogger.new(type: :syslog, formatter: :json)
105
+ @logger
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,80 @@
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ setup_dependencies() {
6
+ sudo apt-get -y update
7
+ sudo apt-get -y autoremove
8
+ sudo apt-get install -y \
9
+ ruby2.0 \
10
+ ruby2.0-dev \
11
+ build-essential \
12
+ git \
13
+ wget \
14
+ vim \
15
+ librabbitmq-dev \
16
+ psmisc \
17
+ curl \
18
+ libcurl4-gnutls-dev \
19
+ python
20
+
21
+ sudo ln -sf /usr/bin/ruby2.0 /usr/bin/ruby && sudo ln -sf /usr/bin/gem2.0 /usr/bin/gem
22
+
23
+ if ! gem list | grep bundler; then
24
+ sudo gem install bundler --no-ri --no-rdoc
25
+ fi
26
+
27
+ # Don't fail because we haven't added github.com's ssh key to our known_hosts:
28
+ cat <<EOF | sudo tee -a /etc/ssh/ssh_config > /dev/null
29
+ Host github.com
30
+ StrictHostKeyChecking no
31
+ EOF
32
+
33
+ sudo gem install eye --no-ri --no-rdoc
34
+ }
35
+
36
+ setup_fluentd() {
37
+ if [ -f /etc/init.d/td-agent ]; then
38
+ echo "fluentd is already installed"
39
+ else
40
+ curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sudo sh
41
+ fi
42
+
43
+ if td-agent-gem list | grep fluent-plugin-elasticsearch; then
44
+ echo "fluent-plugin-elasticsearch is already installed"
45
+ else
46
+ sudo td-agent-gem install fluent-plugin-elasticsearch -v 0.7.0 --no-ri --no-rdoc
47
+ fi
48
+
49
+ if grep "distribot" /etc/rsyslog.conf; then
50
+ echo "syslogd already forwarding events to fluentd"
51
+ else
52
+ echo '!distribot' | sudo tee -a /etc/rsyslog.conf
53
+ echo "*.* @127.0.0.1:42185" | sudo tee -a /etc/rsyslog.conf
54
+ fi
55
+
56
+ sudo cp provision/templates/fluentd.conf /etc/td-agent/td-agent.conf
57
+
58
+ sudo service rsyslog restart
59
+
60
+ # Finally, restart td-agent:
61
+ sudo service td-agent restart
62
+ }
63
+
64
+ setup_dependencies
65
+
66
+ while ! ( echo -e "443\n6379\n5672" | xargs -i nc -w 1 -zv $INFRA_HOST {} ) ; do
67
+ echo "Waiting for infra to come up..."
68
+ sleep 5
69
+ done
70
+
71
+ setup_fluentd
72
+
73
+ cd /var/www/distribot
74
+ bundle
75
+
76
+ echo '--------------------------------------------------------------------------'
77
+ echo '--------------------------------------------------------------------------'
78
+ echo '--------------------------------------------------------------------------'
79
+ echo '--------------------------------------------------------------------------'
80
+ echo '--------------------------------------------------------------------------'
@@ -0,0 +1,27 @@
1
+
2
+
3
+ <source>
4
+ type syslog
5
+ port 42185
6
+ bind 127.0.0.1
7
+ tag syslog
8
+ </source>
9
+
10
+ <match **>
11
+ type copy
12
+ <store>
13
+ type stdout
14
+ </store>
15
+
16
+ <store>
17
+ type elasticsearch
18
+ logstash_format true
19
+ flush_interval 5s
20
+ type_name distribot
21
+
22
+ # We post data to https://<elasticsearch_hostname>/elastic/<index_name>/
23
+ hosts https://infra/elastic/
24
+ index_name distribot
25
+ </store>
26
+
27
+ </match>
@@ -0,0 +1,196 @@
1
+ require 'spec_helper'
2
+
3
+ describe Distribot::BunnyConnector do
4
+
5
+ describe '#initialize(connection_args={})' do
6
+ before do
7
+ @amqp_url = 'amqp://distribot:distribot@172.17.0.2:5672'
8
+ expect(Bunny).to receive(:new).with(@amqp_url) do
9
+ bunny = double('bunny')
10
+ expect(bunny).to receive(:start).ordered
11
+ bunny
12
+ end
13
+ end
14
+
15
+ it 'initializes a new connector' do
16
+ connector = described_class.new(@amqp_url)
17
+ expect(connector).to be_a Distribot::BunnyConnector
18
+ end
19
+
20
+ it 'initializes subscribers as an empty array' do
21
+ connector = described_class.new(@amqp_url)
22
+ expect(connector.subscribers).to eq [ ]
23
+ end
24
+ end
25
+
26
+ describe '#channel' do
27
+ before do
28
+ expect_any_instance_of(described_class).to receive(:setup)
29
+ @connector = described_class.new
30
+ end
31
+ context 'the first time' do
32
+ it 'creates a new channel and returns it' do
33
+ bunny = double('bunny')
34
+ expect(bunny).to receive(:create_channel){ 'a-channel' }
35
+ expect(@connector).to receive(:bunny){ bunny }
36
+ expect(@connector.channel).to eq 'a-channel'
37
+ end
38
+ end
39
+ context 'after the first time' do
40
+ it 'returns the same first channel' do
41
+ bunny = double('bunny')
42
+ expect(bunny).to receive(:create_channel).exactly(1).times{ 'a-channel' }
43
+ expect(@connector).to receive(:bunny){ bunny }
44
+ expect(@connector.channel).to eq 'a-channel'
45
+ expect(@connector.channel).to eq 'a-channel'
46
+ end
47
+ end
48
+ end
49
+
50
+ describe '#queue_exists?(topic)' do
51
+ before do
52
+ expect_any_instance_of(described_class).to receive(:setup)
53
+ @connector = described_class.new(nil)
54
+ @topic = SecureRandom.uuid
55
+ @bunny = double('bunny')
56
+ expect(@connector).to receive(:bunny){ @bunny }
57
+ end
58
+ context 'when the queue' do
59
+ context 'exists' do
60
+ before do
61
+ expect(@bunny).to receive(:queue_exists?).with(@topic){ true }
62
+ end
63
+ it 'returns true' do
64
+ expect(@connector.queue_exists?(@topic)).to be_truthy
65
+ end
66
+ end
67
+ context 'does not exist' do
68
+ before do
69
+ expect(@bunny).to receive(:queue_exists?).with(@topic){ false }
70
+ end
71
+ it 'returns false' do
72
+ expect(@connector.queue_exists?(@topic)).to be_falsey
73
+ end
74
+ end
75
+ end
76
+ end
77
+
78
+ describe '#subscribe(topic, options={}, &block)' do
79
+ before do
80
+ expect_any_instance_of(described_class).to receive(:setup)
81
+ @connector = described_class.new(nil)
82
+ @topic = SecureRandom.uuid
83
+ expect_any_instance_of(Distribot::Subscription).to receive(:start).with(@topic, {}) do |&block|
84
+ block.call( id: 'hello' )
85
+ end
86
+ end
87
+ context 'when options[:solo]' do
88
+ context 'is truthy' do
89
+ it 'first calls setup, then subscribes to the topic, and calls the block when a message is received' do
90
+ expect(@connector).to receive(:setup)
91
+ @connector.subscribe(@topic, solo: true) do |msg|
92
+ @id = msg[:id]
93
+ end
94
+ expect(@id).to eq 'hello'
95
+ end
96
+ end
97
+ context 'is falsey' do
98
+ it 'subscribes to the topic, and calls the block when a message is received' do
99
+ @connector.subscribe(@topic) do |msg|
100
+ @id = msg[:id]
101
+ end
102
+ expect(@id).to eq 'hello'
103
+ end
104
+ end
105
+ end
106
+ end
107
+
108
+ describe '#subscribe_multi(topic, options={}, &block)' do
109
+ before do
110
+ expect_any_instance_of(described_class).to receive(:setup)
111
+ @connector = described_class.new(nil)
112
+ @topic = SecureRandom.uuid
113
+ expect_any_instance_of(Distribot::MultiSubscription).to receive(:start).with(@topic, {}) do |&block|
114
+ block.call( id: 'hello' )
115
+ end
116
+ end
117
+ it 'subscribes to the topic, and calls the block when a message is received' do
118
+ @connector.subscribe_multi(@topic) do |msg|
119
+ @id = msg[:id]
120
+ end
121
+ expect(@id).to eq 'hello'
122
+ end
123
+ end
124
+
125
+ describe '#publish(topic, message)' do
126
+ before do
127
+ @topic = SecureRandom.uuid
128
+ expect_any_instance_of(described_class).to receive(:setup)
129
+ @connector = described_class.new
130
+ @channel = double('channel')
131
+ expect(@channel).to receive(:queue).with(@topic, auto_delete: false, durable: true) do
132
+ queue = double('queue')
133
+ expect(queue).to receive(:name){ @topic }
134
+ queue
135
+ end
136
+ expect(@channel).to receive(:default_exchange) do
137
+ exchange = double('exchange')
138
+ expect(exchange).to receive(:publish).with( '{"hello":"world"}', routing_key: @topic)
139
+ exchange
140
+ end
141
+ expect(@connector).to receive(:channel).exactly(2).times{ @channel }
142
+ end
143
+ it 'publishes the message' do
144
+ @connector.publish(@topic, {hello: :world})
145
+ end
146
+ end
147
+
148
+ describe '#broadcast(topic, message)' do
149
+ before do
150
+ @topic = SecureRandom.uuid
151
+ expect_any_instance_of(described_class).to receive(:setup)
152
+ @connector = described_class.new
153
+ @channel = double('channel')
154
+ expect(@channel).to receive(:fanout).with(@topic) do
155
+ exchange = double('exchange')
156
+ expect(exchange).to receive(:publish).with('{"hello":"world"}', routing_key: @topic)
157
+ exchange
158
+ end
159
+ expect(@connector).to receive(:channel){ @channel }
160
+ end
161
+ it 'broadcasts a message on a fanout exchange' do
162
+ @connector.broadcast(@topic, {hello: :world})
163
+ end
164
+ end
165
+
166
+ describe '#stubbornly' do
167
+ context 'when the block' do
168
+ before do
169
+ expect_any_instance_of(described_class).to receive(:setup)
170
+ end
171
+ context 'raises an error' do
172
+ it 'keeps trying forever, until it stops raising an error' do
173
+ @return_value = SecureRandom.uuid
174
+ thing = described_class.new
175
+ @max_tries = 3
176
+ @total_tries = 0
177
+ expect(thing.send(:stubbornly, :foo){
178
+ if @total_tries >= @max_tries
179
+ @return_value
180
+ else
181
+ @total_tries += 1
182
+ raise Timeout::Error.new
183
+ end
184
+ }).to eq @return_value
185
+ end
186
+ end
187
+ context 'does not raise an error' do
188
+ it 'returns the result of the block' do
189
+ @return_value = SecureRandom.uuid
190
+ thing = described_class.new
191
+ expect(thing.send(:stubbornly, :foo){ @return_value }).to eq @return_value
192
+ end
193
+ end
194
+ end
195
+ end
196
+ end
@@ -0,0 +1,34 @@
1
+ require 'spec_helper'
2
+
3
+ describe Distribot::ConnectionSharer do
4
+ describe '#initialize(bunny)' do
5
+ before do
6
+ @bunny = SecureRandom.uuid
7
+ @sharer = described_class.new(@bunny)
8
+ end
9
+ it 'sets self.bunny' do
10
+ expect(@sharer.bunny).to eq @bunny
11
+ end
12
+ end
13
+
14
+ describe '#channel' do
15
+ before do
16
+ @bunny = double('bunny')
17
+ expect(@bunny).to receive(:create_channel){ SecureRandom.uuid }
18
+ @sharer = described_class.new(@bunny)
19
+ end
20
+ context 'the first time' do
21
+ it 'creates a new channel and stores it' do
22
+ expect(SecureRandom).to receive(:uuid){ 'your-channel' }
23
+ expect(@sharer.channel).to eq 'your-channel'
24
+ end
25
+ end
26
+ context 'each subsequent time' do
27
+ it 'returns the original channel' do
28
+ expect(SecureRandom).to receive(:uuid).exactly(1).times.and_call_original
29
+ first_channel = @sharer.channel
30
+ expect(@sharer.channel).to eq first_channel
31
+ end
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,63 @@
1
+
2
+ require 'spec_helper'
3
+
4
+ describe Distribot::Connector do
5
+
6
+ describe '#initialize(connection_args={})' do
7
+ before do
8
+ @amqp_url = 'amqp://distribot:distribot@172.17.0.2:5672'
9
+ expect(Bunny).to receive(:new).with(@amqp_url) do
10
+ bunny = double('bunny')
11
+ expect(bunny).to receive(:start).ordered
12
+ expect(bunny).to receive(:create_channel).ordered do
13
+ channel = double('channel')
14
+ expect(channel).to receive(:prefetch).with(1)
15
+ channel
16
+ end
17
+ bunny
18
+ end
19
+ end
20
+
21
+ it 'initializes a new connector' do
22
+ connector = described_class.new(@amqp_url)
23
+ expect(connector).to be_a Distribot::Connector
24
+ end
25
+ end
26
+
27
+ describe '#queues' do
28
+ before do
29
+ expect_any_instance_of(described_class).to receive(:setup)
30
+ @connector = described_class.new
31
+ @queues = %w(
32
+ distribot.flow.created
33
+ distribot.flow.finished
34
+ distribot.flow.handler.CheapWorker.1.0.0.enumerate
35
+ distribot.flow.handler.CheapWorker.1.0.0.tasks
36
+ distribot.flow.handler.FastWorker.1.0.0.enumerate
37
+ distribot.flow.handler.FastWorker.1.0.0.tasks
38
+ distribot.flow.handler.ForeignWorker.1.0.0.enumerate
39
+ distribot.flow.handler.ForeignWorker.1.0.0.tasks
40
+ distribot.flow.handler.GoodWorker.1.0.0.enumerate
41
+ distribot.flow.handler.GoodWorker.1.0.0.tasks
42
+ distribot.flow.handler.HardWorker.1.0.0.enumerate
43
+ distribot.flow.handler.HardWorker.1.0.0.tasks
44
+ distribot.flow.handler.SlowWorker.1.0.0.enumerate
45
+ distribot.flow.handler.SlowWorker.1.0.0.tasks
46
+ distribot.flow.handler.finished
47
+ distribot.flow.phase.finished
48
+ distribot.flow.phase.started
49
+ distribot.flow.task.finished
50
+ )
51
+ @queues_json = @queues.to_a.map{|name| {name: name} }.to_json
52
+ Wrest.logger = Logger.new('/dev/null')
53
+ stub_request(:get, "http://localhost:15672/api/queues").
54
+ with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
55
+ to_return(:status => 200, :body => @queues_json, :headers => {'Content-Type' => 'application/json'})
56
+ end
57
+ it 'returns all the queues from /api/queues on rabbitmq' do
58
+ result = @connector.queues
59
+ expect(result).to eq(@queues)
60
+ end
61
+ end
62
+
63
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+ describe Distribot::FlowCreatedHandler do
4
+ describe 'definition' do
5
+ it 'subscribes to the correct queue' do
6
+ expect(Distribot::Handler.queue_for(described_class)).to eq 'distribot.flow.created'
7
+ end
8
+ it 'declares a valid handler' do
9
+ expect(Distribot::Handler.handler_for(described_class)).to eq :callback
10
+ end
11
+ it 'has a method matching the handler name' do
12
+ expect(Distribot).to receive(:subscribe)
13
+ expect(described_class.new).to respond_to :callback
14
+ end
15
+ end
16
+
17
+ describe '#callback' do
18
+ before do
19
+ @flow_id = SecureRandom.uuid
20
+ expect(Distribot::Flow).to receive(:find).with(@flow_id) do
21
+ flow = double('flow')
22
+ expect(flow).to receive(:next_phase){'phase2'}
23
+ expect(flow).to receive(:transition_to!).with('phase2'){ true }
24
+ flow
25
+ end
26
+ expect(Distribot).to receive(:subscribe)
27
+ end
28
+ it 'transitions to the next phase' do
29
+ expect(described_class.new.callback(flow_id: @flow_id)).to be_truthy
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,32 @@
1
+ require 'spec_helper'
2
+
3
+ describe Distribot::FlowFinishedHandler do
4
+ describe 'definition' do
5
+ it 'subscribes to the distribot.flow.finished queue' do
6
+ expect(Distribot::Handler.queue_for(described_class)).to eq 'distribot.flow.finished'
7
+ end
8
+ it 'declares a valid handler' do
9
+ expect(Distribot::Handler.handler_for(described_class)).to eq :callback
10
+ end
11
+ it 'has a method matching the handler name' do
12
+ expect(Distribot).to receive(:subscribe)
13
+ expect(described_class.new).to respond_to :callback
14
+ end
15
+ end
16
+
17
+ describe '#callback' do
18
+ before do
19
+ @message = {
20
+ flow_id: 'xxx'
21
+ }
22
+ redis = double('redis')
23
+ expect(redis).to receive(:decr).with('distribot.flows.running')
24
+ expect(redis).to receive(:srem).with('distribot.flows.active', @message[:flow_id])
25
+ expect(Distribot).to receive(:redis).exactly(2).times{ redis }
26
+ expect(Distribot).to receive(:subscribe)
27
+ end
28
+ it 'decrements the running tally of how many flows are currently in process' do
29
+ described_class.new.callback(@message)
30
+ end
31
+ end
32
+ end