mlanett-hive 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (63) hide show
  1. data/.autotest +13 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +16 -0
  5. data/Guardfile +6 -0
  6. data/README +9 -0
  7. data/Rakefile +11 -0
  8. data/bin/hive +37 -0
  9. data/demo/demo +36 -0
  10. data/demo/demo.rb +30 -0
  11. data/demo/demo3 +36 -0
  12. data/demo/job1.rb +31 -0
  13. data/demo/job2.rb +42 -0
  14. data/demo/job3.rb +44 -0
  15. data/demo/populate.rb +22 -0
  16. data/hive.gemspec +21 -0
  17. data/lib/hive.rb +42 -0
  18. data/lib/hive/checker.rb +51 -0
  19. data/lib/hive/configuration.rb +251 -0
  20. data/lib/hive/idler.rb +81 -0
  21. data/lib/hive/key.rb +48 -0
  22. data/lib/hive/lifecycle_observer.rb +25 -0
  23. data/lib/hive/log.rb +29 -0
  24. data/lib/hive/messager.rb +217 -0
  25. data/lib/hive/mocks/storage.rb +112 -0
  26. data/lib/hive/monitor.rb +57 -0
  27. data/lib/hive/policy.rb +68 -0
  28. data/lib/hive/pool.rb +180 -0
  29. data/lib/hive/redis/storage.rb +145 -0
  30. data/lib/hive/registry.rb +123 -0
  31. data/lib/hive/squiggly.rb +20 -0
  32. data/lib/hive/trace.rb +5 -0
  33. data/lib/hive/utilities/airbrake_observer.rb +26 -0
  34. data/lib/hive/utilities/hoptoad_observer.rb +26 -0
  35. data/lib/hive/utilities/log_observer.rb +40 -0
  36. data/lib/hive/utilities/observeable.rb +18 -0
  37. data/lib/hive/utilities/observer_base.rb +59 -0
  38. data/lib/hive/utilities/process.rb +82 -0
  39. data/lib/hive/utilities/resolver.rb +12 -0
  40. data/lib/hive/utilities/signal_hook.rb +47 -0
  41. data/lib/hive/utilities/storage_base.rb +41 -0
  42. data/lib/hive/version.rb +3 -0
  43. data/lib/hive/worker.rb +162 -0
  44. data/spec/checker_spec.rb +20 -0
  45. data/spec/configuration_spec.rb +50 -0
  46. data/spec/helper.rb +33 -0
  47. data/spec/idler_spec.rb +58 -0
  48. data/spec/key_spec.rb +41 -0
  49. data/spec/messager_spec.rb +131 -0
  50. data/spec/mocks/storage_spec.rb +108 -0
  51. data/spec/monitor_spec.rb +15 -0
  52. data/spec/policy_spec.rb +43 -0
  53. data/spec/pool_spec.rb +119 -0
  54. data/spec/redis/storage_spec.rb +133 -0
  55. data/spec/registry_spec.rb +52 -0
  56. data/spec/support/jobs.rb +68 -0
  57. data/spec/support/redis.rb +22 -0
  58. data/spec/support/timing.rb +32 -0
  59. data/spec/utilities/observer_base_spec.rb +50 -0
  60. data/spec/utilities/process_spec.rb +17 -0
  61. data/spec/worker_spec.rb +121 -0
  62. data/unused/times.rb +45 -0
  63. metadata +148 -0
@@ -0,0 +1,112 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ class Hive::Mocks::Storage
4
+
5
+ def initialize
6
+ @storage = {}
7
+ end
8
+
9
+ def reconnect_after_fork
10
+ # nop
11
+ end
12
+
13
+ def to_s
14
+ "#{self.class.name}()"
15
+ end
16
+
17
+ # Simple values
18
+
19
+ def put( key, value )
20
+ @storage[key] = value
21
+ end
22
+
23
+ def get( key )
24
+ @storage[key]
25
+ end
26
+
27
+ def del( key )
28
+ @storage.delete( key )
29
+ end
30
+
31
+ # Sets
32
+
33
+ def set_add( key, value )
34
+ @storage[key] ||= []
35
+ @storage[key] << value unless @storage[key].member?(value)
36
+ end
37
+
38
+ def set_size( key )
39
+ (@storage[key] || [] ).size
40
+ end
41
+
42
+ def set_remove( key, value )
43
+ (@storage[key] || [] ).delete( value )
44
+ end
45
+
46
+ def set_member?( key, value )
47
+ (@storage[key] || []).member?( value )
48
+ end
49
+
50
+ def set_get_all( key )
51
+ @storage[key] || []
52
+ end
53
+
54
+ # Maps
55
+
56
+ def map_set( key, name, value )
57
+ @storage[key] ||= {}
58
+ @storage[key][name] = value
59
+ end
60
+
61
+ def map_get( key, name )
62
+ (@storage[key] || {}) [name]
63
+ end
64
+
65
+ def map_get_all_keys( key )
66
+ (@storage[key] || {}).keys
67
+ end
68
+
69
+ def map_size( key )
70
+ (@storage[key] || {} ).size
71
+ end
72
+
73
+ def map_del( key )
74
+ @storage.delete( key )
75
+ end
76
+
77
+ # Priority Queue
78
+
79
+ def queue_add( queue_name, item, score )
80
+ queue = @storage[queue_name] ||= []
81
+ queue << [ item, score ]
82
+ queue.sort_by! { |it| it.last }
83
+ end
84
+
85
+ # pop the lowest item from the queue IFF it scores <= max_score
86
+ def queue_pop( queue_name, max_score = Time.now.to_i )
87
+ queue = @storage[queue_name] || []
88
+ return nil if queue.size == 0
89
+ if queue.first.last <= max_score then
90
+ queue.shift.first
91
+ else
92
+ nil
93
+ end
94
+ end
95
+
96
+ def queue_pop_sync( queue_name, max_score = Time.now.to_i, options = {} )
97
+ timeout = options[:timeout] || 1
98
+ deadline = Time.now.to_f + timeout
99
+
100
+ loop do
101
+ result = queue_pop( queue_name, max_score )
102
+ return result if result
103
+
104
+ raise Timeout::Error if Time.now.to_f > deadline
105
+ end
106
+ end
107
+
108
+ def queue_del( queue_name )
109
+ @storage.delete( queue_name )
110
+ end
111
+
112
+ end # Hive::Mocks::Storage
@@ -0,0 +1,57 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ class Hive::Monitor
4
+
5
+ include Hive::Log
6
+
7
+ attr :pools
8
+
9
+ def initialize( configuration )
10
+ @pools = configuration.policies.map do |kind,policy|
11
+ pool = Hive::Pool.new( kind, policy )
12
+ end
13
+ end
14
+
15
+
16
+ def monitor
17
+ status = {}
18
+
19
+ job = ->() do
20
+ changed = false
21
+ pools.each do |pool|
22
+
23
+ log pool.name
24
+ previous = status[pool.name]
25
+ current = pool.synchronize log: true
26
+
27
+ if previous != current then
28
+ status[pool.name] = current
29
+ changed = true
30
+ end
31
+
32
+ end
33
+ changed
34
+ end
35
+
36
+ job = Hive::Idler.new( job, min_sleep: 1, max_sleep: 10 )
37
+
38
+ ok = true
39
+ trap("TERM") { ok = false }
40
+ while ok do
41
+ job.call
42
+ end
43
+ end # monitor
44
+
45
+ def stop_all
46
+ pools.each do |pool|
47
+ pool.stop_all
48
+ end
49
+ end
50
+
51
+ def restart
52
+ pools.each do |pool|
53
+ pool.restart
54
+ end
55
+ end
56
+
57
+ end
@@ -0,0 +1,68 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "ostruct"
4
+
5
+ class Hive::Policy
6
+
7
+ DEFAULTS = {
8
+ pool_min_workers: 1,
9
+ pool_max_workers: 10,
10
+ worker_idle_max_sleep: 64.0,
11
+ worker_idle_min_sleep: 0.125,
12
+ worker_idle_spin_down: 900,
13
+ worker_none_spin_up: 86400,
14
+ worker_max_jobs: 100, # a worker should automatically exit after this many jobs
15
+ worker_max_lifetime: 1000, # a worker should automatically exit after this time
16
+ worker_late: 10, # a worker is overdue after this time with no heartbeat
17
+ worker_hung: 100, # a worker will be killed after this time
18
+ storage: :mock,
19
+ observers: []
20
+ }
21
+
22
+ class Instance
23
+
24
+ # including options[:policy] will merge over these options
25
+ def initialize( options = {} )
26
+ if options[:policy] then
27
+ policy = options.delete(:policy)
28
+ defaults = policy.dup
29
+ else
30
+ defaults = DEFAULTS
31
+ end
32
+
33
+ options = Hash[ options.map { |k,v| [ k.to_sym, v ] } ] # poor man's symbolize keys
34
+ @options = defaults.merge( options )
35
+ end
36
+
37
+ def storage
38
+ Hive::Utilities::StorageBase.resolve @options[:storage]
39
+ end
40
+
41
+ def method_missing( symbol, *arguments )
42
+ @options[symbol.to_sym]
43
+ end
44
+
45
+ def dup
46
+ @options.dup
47
+ end
48
+
49
+ def before_fork
50
+ (before_forks || []).each { |f| f.call }
51
+ end
52
+
53
+ def after_fork
54
+ (after_forks || []).each { |f| f.call }
55
+ end
56
+
57
+ end # Instance
58
+
59
+ class << self
60
+
61
+ def resolve( options = {} )
62
+ # this will dup either an Instance or a Hash
63
+ Hive::Policy::Instance.new(options.dup)
64
+ end
65
+
66
+ end # class
67
+
68
+ end # Hive::Policy
data/lib/hive/pool.rb ADDED
@@ -0,0 +1,180 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ =begin
4
+
5
+ A pool is a collection of workers, each of which is a separate process.
6
+ All workers are of the same kind (class).
7
+
8
+ =end
9
+
10
+ class Hive::Pool
11
+
12
+ include Hive::Log
13
+
14
+ attr :kind # job class
15
+ attr :name
16
+ attr :policy
17
+ attr :registry
18
+ attr :storage # where to store worker details
19
+
20
+ def initialize( kind, policy_prototype = {} )
21
+ if kind.kind_of?(Array) then
22
+ kind, policy_prototype = kind.first, kind.last
23
+ end
24
+ @kind = kind
25
+ @policy = Hive::Policy.resolve(policy_prototype) or raise
26
+ @name = @policy.name || kind.name or raise Hive::ConfigurationError, "Pool or Job must have a name"
27
+ @storage = policy.storage
28
+ @registry = Hive::Registry.new( name, storage )
29
+
30
+ # type checks
31
+ policy.pool_min_workers
32
+ registry.workers
33
+ end
34
+
35
+
36
+ # @param options[:log] can be true
37
+ # @returns the checked worker lists
38
+ def synchronize( options = {} )
39
+ do_log = options.delete(:log)
40
+ raise if options.size > 0
41
+
42
+ checklist = registry.checked_workers( policy )
43
+ live_count = checklist.live.size
44
+
45
+ if do_log then
46
+ check_live_workers( checklist )
47
+ check_late_workers( checklist )
48
+ check_hung_workers( checklist )
49
+ check_dead_workers( checklist )
50
+ end
51
+
52
+ if (need = policy.pool_min_workers - live_count) > 0 then
53
+ # launch workers
54
+ need.times do
55
+ spawn wait: true
56
+ end
57
+
58
+ elsif (excess = live_count - policy.pool_max_workers) > 0 then
59
+ # spin down some workers
60
+ # try to find LOCAL workers to spin down first
61
+ locals = checklist.live.select { |k| k.host == Hive::Key.local_host }
62
+ if locals.size > 0 then
63
+ reap locals.first, wait: true
64
+ else
65
+ reap checklist.live.first, wait: true
66
+ end
67
+ end
68
+
69
+ checklist = registry.checked_workers( policy )
70
+ end
71
+
72
+
73
+ def mq
74
+ @mq ||= begin
75
+ key = Hive::Key.new( "#{name}-pool", Process.pid )
76
+ me = Hive::Messager.new storage, my_address: key
77
+ end
78
+ end
79
+
80
+
81
+ # tell all workers to quit
82
+ def stop_all
83
+ checklist = registry.checked_workers( policy )
84
+ checklist.live.each { |key| reap(key) }
85
+ checklist.late.each { |key| reap(key) }
86
+ checklist.hung.each { |key| reap(key) }
87
+ checklist.dead.each { |key| reap(key) }
88
+ end
89
+
90
+
91
+ # this really should be protected but it's convenient to be able to force a spawn
92
+ # param options[:wait] can true to wait until after the process is spawned
93
+ def spawn( options = {} )
94
+ wait = options.delete(:wait)
95
+ raise if options.size > 0
96
+
97
+ if ! wait then
98
+ Hive::Worker.spawn kind, registry: registry, policy: policy, name: name
99
+ return
100
+ end
101
+
102
+ before = registry.checked_workers( policy ).live
103
+
104
+ Hive::Worker.spawn kind, registry: registry, policy: policy, name: name
105
+
106
+ Hive::Idler.wait_until( 10 ) do
107
+ after = registry.checked_workers( policy ).live
108
+ diff = ( after - before ).select { |k| k.host == Hive::Key.local_host }
109
+ diff.size > 0
110
+ end
111
+ end
112
+
113
+
114
+ # shut down a worker
115
+ def reap( key, options = {} )
116
+ wait = options.delete(:wait)
117
+ raise if options.size > 0
118
+
119
+ if key.host == Hive::Key.local_host then
120
+ ::Process.kill( "TERM", key.pid )
121
+ Hive::Utilities::Process.wait_and_terminate key.pid, timeout: 10
122
+ else
123
+ mq.send "Quit", to: key
124
+ end
125
+
126
+ if wait then
127
+ Hive::Idler.wait_until( 10 ) do
128
+ live = registry.checked_workers( policy ).live
129
+ ! live.member? key
130
+ end
131
+ end
132
+ end
133
+
134
+ # ----------------------------------------------------------------------------
135
+ protected
136
+ # ----------------------------------------------------------------------------
137
+
138
+ def check_live_workers( checked )
139
+ if live = checked.live and live.size > 0 then
140
+ log "Live worker count #{live.size}; members: #{live.inspect}"
141
+ live.size
142
+ else
143
+ 0
144
+ end
145
+ end
146
+
147
+
148
+ def check_late_workers( checked )
149
+ if late = checked.late and late.size > 0 then
150
+ log "Late worker count #{late.size}; members: #{late.inspect}"
151
+ late.size
152
+ else
153
+ 0
154
+ end
155
+ end
156
+
157
+
158
+ def check_hung_workers( checked )
159
+ if hung = checked.hung and hung.size > 0 then
160
+ log "Hung worker count #{hung.size}"
161
+ hung.each do |key|
162
+ log "Killing #{key}"
163
+ Hive::Utilities::Process.wait_and_terminate( key.pid )
164
+ registry.unregister(key)
165
+ end
166
+ end
167
+ 0
168
+ end
169
+
170
+
171
+ def check_dead_workers( checked )
172
+ if dead = checked.dead and dead.size > 0 then
173
+ log "Dead worker count #{dead.size}; members: #{dead.inspect}"
174
+ dead.size
175
+ else
176
+ 0
177
+ end
178
+ end
179
+
180
+ end # Hive::Pool
@@ -0,0 +1,145 @@
1
+ # -*- encoding: utf-8 -*-
2
+
3
+ require "redis"
4
+ require "redis-namespace"
5
+ require "timeout"
6
+
7
+ class Hive::Redis::Storage
8
+
9
+ def initialize( redis = nil )
10
+ self.redis = redis if redis
11
+ end
12
+
13
+ def reconnect_after_fork
14
+ redis.client.disconnect
15
+ end
16
+
17
+ def to_s
18
+ "#{self.class.name}(#{redis.inspect})"
19
+ end
20
+
21
+ # Simple values
22
+
23
+ def put( key, value )
24
+ redis.set( key, value )
25
+ end
26
+
27
+ def get( key )
28
+ redis.get( key )
29
+ end
30
+
31
+ def del( key )
32
+ redis.del( key )
33
+ end
34
+
35
+ # Sets
36
+
37
+ def set_add( set_name, value )
38
+ redis.sadd( set_name, value )
39
+ end
40
+
41
+ def set_size( set_name )
42
+ redis.scard( set_name )
43
+ end
44
+
45
+ def set_remove( set_name, value )
46
+ redis.srem( set_name, value )
47
+ end
48
+
49
+ def set_get_all( set_name )
50
+ redis.smembers( set_name )
51
+ end
52
+
53
+ def set_member?( set_name, value )
54
+ redis.sismember( set_name, value )
55
+ end
56
+
57
+ # Priority Queue
58
+
59
+ def queue_add( queue_name, item, score = Time.now.to_i )
60
+ score = score.to_f
61
+ begin
62
+ redis.zadd( queue_name, score, item )
63
+ rescue Exception => x
64
+ raise x, "Failed zadd( #{queue_name.inspect}, #{score.inspect}, #{item.inspect} ) because of an error: #{x.message}", x.backtrace
65
+ end
66
+ end
67
+
68
+ # pop the lowest item from the queue IFF it scores <= max_score
69
+ def queue_pop( queue_name, max_score = Time.now.to_i )
70
+ # Option 1: zrange, check score, accept or discard
71
+ # Option 2: zrangebyscore with limit, then zremrangebyrank
72
+
73
+ redis.watch( queue_name )
74
+ it = redis.zrangebyscore( queue_name, 0, max_score, limit: [0,1] ).first
75
+ if it then
76
+ ok = redis.multi { |r| r.zremrangebyrank( queue_name, 0, 0 ) }
77
+ it = nil if ! ok
78
+ else
79
+ redis.unwatch
80
+ end
81
+ it
82
+ end
83
+
84
+ def queue_pop_sync( queue_name, max_score = Time.now.to_i, options = {} )
85
+ timeout = options[:timeout] || 1
86
+ deadline = Time.now.to_f + timeout
87
+
88
+ loop do
89
+ result = queue_pop( queue_name, max_score )
90
+ return result if result
91
+
92
+ raise Timeout::Error if Time.now.to_f > deadline
93
+ end
94
+ end
95
+
96
+ def queue_del( queue_name )
97
+ redis.del( queue_name )
98
+ end
99
+
100
+ # Maps
101
+
102
+ def map_set( map_name, key, value )
103
+ redis.hset( map_name, key, value )
104
+ end
105
+
106
+ def map_get( map_name, key )
107
+ redis.hget( map_name, key )
108
+ end
109
+
110
+ def map_get_all_keys( map_name )
111
+ redis.hkeys( map_name )
112
+ end
113
+
114
+ def map_size( map_name )
115
+ redis.hlen( map_name )
116
+ end
117
+
118
+ def map_del( map_name, key )
119
+ redis.hdel( map_name, key )
120
+ end
121
+
122
+ # ----------------------------------------------------------------------------
123
+ # Redis
124
+ # ----------------------------------------------------------------------------
125
+
126
+ # @param redis_client can only be set once
127
+ def redis=( redis_or_options )
128
+ raise Hive::ConfigurationError if @redis
129
+
130
+ case redis_or_options
131
+ when Hash
132
+ options = redis_or_options.dup
133
+ namespace = options.delete(:namespace)
134
+ @redis = Redis.connect(options)
135
+ @redis = Redis::Namespace.new( namespace, redis: @redis ) if namespace
136
+ else
137
+ @redis = redis_or_options
138
+ end
139
+ end
140
+
141
+ def redis
142
+ @redis ||= ::Redis.connect( url: "redis://127.0.0.1:6379/1" )
143
+ end
144
+
145
+ end # Hive::Redis::Storage