mlanett-hive 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +13 -0
- data/.gitignore +6 -0
- data/.rspec +1 -0
- data/Gemfile +16 -0
- data/Guardfile +6 -0
- data/README +9 -0
- data/Rakefile +11 -0
- data/bin/hive +37 -0
- data/demo/demo +36 -0
- data/demo/demo.rb +30 -0
- data/demo/demo3 +36 -0
- data/demo/job1.rb +31 -0
- data/demo/job2.rb +42 -0
- data/demo/job3.rb +44 -0
- data/demo/populate.rb +22 -0
- data/hive.gemspec +21 -0
- data/lib/hive.rb +42 -0
- data/lib/hive/checker.rb +51 -0
- data/lib/hive/configuration.rb +251 -0
- data/lib/hive/idler.rb +81 -0
- data/lib/hive/key.rb +48 -0
- data/lib/hive/lifecycle_observer.rb +25 -0
- data/lib/hive/log.rb +29 -0
- data/lib/hive/messager.rb +217 -0
- data/lib/hive/mocks/storage.rb +112 -0
- data/lib/hive/monitor.rb +57 -0
- data/lib/hive/policy.rb +68 -0
- data/lib/hive/pool.rb +180 -0
- data/lib/hive/redis/storage.rb +145 -0
- data/lib/hive/registry.rb +123 -0
- data/lib/hive/squiggly.rb +20 -0
- data/lib/hive/trace.rb +5 -0
- data/lib/hive/utilities/airbrake_observer.rb +26 -0
- data/lib/hive/utilities/hoptoad_observer.rb +26 -0
- data/lib/hive/utilities/log_observer.rb +40 -0
- data/lib/hive/utilities/observeable.rb +18 -0
- data/lib/hive/utilities/observer_base.rb +59 -0
- data/lib/hive/utilities/process.rb +82 -0
- data/lib/hive/utilities/resolver.rb +12 -0
- data/lib/hive/utilities/signal_hook.rb +47 -0
- data/lib/hive/utilities/storage_base.rb +41 -0
- data/lib/hive/version.rb +3 -0
- data/lib/hive/worker.rb +162 -0
- data/spec/checker_spec.rb +20 -0
- data/spec/configuration_spec.rb +50 -0
- data/spec/helper.rb +33 -0
- data/spec/idler_spec.rb +58 -0
- data/spec/key_spec.rb +41 -0
- data/spec/messager_spec.rb +131 -0
- data/spec/mocks/storage_spec.rb +108 -0
- data/spec/monitor_spec.rb +15 -0
- data/spec/policy_spec.rb +43 -0
- data/spec/pool_spec.rb +119 -0
- data/spec/redis/storage_spec.rb +133 -0
- data/spec/registry_spec.rb +52 -0
- data/spec/support/jobs.rb +68 -0
- data/spec/support/redis.rb +22 -0
- data/spec/support/timing.rb +32 -0
- data/spec/utilities/observer_base_spec.rb +50 -0
- data/spec/utilities/process_spec.rb +17 -0
- data/spec/worker_spec.rb +121 -0
- data/unused/times.rb +45 -0
- metadata +148 -0
data/.autotest
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
# (this file is loaded automatically by autotest)
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
|
4
|
+
# Bundler support
|
5
|
+
# @see https://www.relishapp.com/rspec/rspec-core/docs/autotest
|
6
|
+
require "autotest/bundler"
|
7
|
+
|
8
|
+
# ... add the following line after all other requires in your ~/.autotest:
|
9
|
+
begin
|
10
|
+
require "autotest/fsevent"
|
11
|
+
rescue LoadError => x
|
12
|
+
raise x if RUBY_PLATFORM =~ /darwin/
|
13
|
+
end
|
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--color
|
data/Gemfile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in hive.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
group :development, :test do
|
7
|
+
gem "guard-rspec"
|
8
|
+
gem "rb-fsevent" # for guard
|
9
|
+
gem "growl_notify" # for guard
|
10
|
+
gem "rspec"
|
11
|
+
gem "ruby-debug19", require: false
|
12
|
+
end
|
13
|
+
|
14
|
+
group :test do
|
15
|
+
gem "simplecov", require: false
|
16
|
+
end
|
data/Guardfile
ADDED
data/README
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rspec/core/rake_task"
|
3
|
+
|
4
|
+
# the minitest way
|
5
|
+
# task :test do
|
6
|
+
# require_relative "./spec/helper"
|
7
|
+
# FileList["spec/**/*_{spec}.rb"].each { |f| load(f) }
|
8
|
+
# end
|
9
|
+
|
10
|
+
RSpec::Core::RakeTask.new(:spec)
|
11
|
+
task :default => :spec
|
data/bin/hive
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
=begin
|
4
|
+
|
5
|
+
Usage:
|
6
|
+
hive [options] [check] | monitor | restart | status | stop
|
7
|
+
|
8
|
+
e.g.
|
9
|
+
hive --dry-run --verbose --env production --name it --chdir /tmp/it check
|
10
|
+
hive /opt/rails/it/current/config/it monitor
|
11
|
+
Options must preceed the command.
|
12
|
+
|
13
|
+
Note: the pid file depends on chdir, env, and name, so you have to specify them for any command to work.
|
14
|
+
They can be specified in a configuration file.
|
15
|
+
|
16
|
+
=end
|
17
|
+
|
18
|
+
# We don't use Bundler because that is up to the client.
|
19
|
+
# The client can use 'bundle exec hive' if desired.
|
20
|
+
File.expand_path(File.dirname(__FILE__)+"/../lib").tap { |d| $:.unshift(d) if ! $:.member?(d) }
|
21
|
+
require "hive"
|
22
|
+
|
23
|
+
my = Hive::Configuration.parse(ARGV)
|
24
|
+
|
25
|
+
# TODO use my.options_for_daemon_spawn
|
26
|
+
# TODO use my.args_for_daemon_spawn
|
27
|
+
|
28
|
+
if ! my.dry_run then
|
29
|
+
case ARGV.first
|
30
|
+
when "stop"
|
31
|
+
Hive::Monitor.new(my).stop_all
|
32
|
+
when "monitor"
|
33
|
+
Hive::Monitor.new(my).monitor
|
34
|
+
else
|
35
|
+
Kernel.abort "Unknown command #{ARGV.first}"
|
36
|
+
end
|
37
|
+
end
|
data/demo/demo
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/usr/bin/env ruby bin/hive
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
|
4
|
+
# This file is evaluated in the context of a Hive::Configuration instance.
|
5
|
+
# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
|
6
|
+
# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
|
7
|
+
|
8
|
+
add_path File.dirname(__FILE__)
|
9
|
+
require "job1"
|
10
|
+
|
11
|
+
# best to pass these on the command line
|
12
|
+
# set_env "development" if ! env
|
13
|
+
# set_name "demo"
|
14
|
+
# chdir "/tmp/demo"
|
15
|
+
|
16
|
+
set_defaults observers: [ :log ] unless env == "test" # No noise when testing please.
|
17
|
+
|
18
|
+
set_defaults(
|
19
|
+
pool_min_workers: 1,
|
20
|
+
worker_late: 10,
|
21
|
+
worker_hung: 100,
|
22
|
+
batchsize: 100,
|
23
|
+
worker_max_lifetime: 3600,
|
24
|
+
observers: [],
|
25
|
+
storage: :redis
|
26
|
+
)
|
27
|
+
|
28
|
+
add_pool Job1,
|
29
|
+
worker_late: 60,
|
30
|
+
pool_min_workers: 1,
|
31
|
+
pool_max_workers: 10
|
32
|
+
|
33
|
+
after_fork do
|
34
|
+
# reset ActiveRecord and other pools
|
35
|
+
puts "Forked!"
|
36
|
+
end
|
data/demo/demo.rb
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
# This file is evaluated in the context of a Hive::Configuration instance.
|
4
|
+
# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
|
5
|
+
# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
|
6
|
+
|
7
|
+
add_path File.dirname(__FILE__)
|
8
|
+
require "job1"
|
9
|
+
|
10
|
+
set_env "development" if ! env
|
11
|
+
set_name "demo"
|
12
|
+
chdir "/tmp/demo"
|
13
|
+
|
14
|
+
set_defaults(
|
15
|
+
pool_min_workers: 1,
|
16
|
+
worker_late: 10,
|
17
|
+
worker_hung: 100,
|
18
|
+
batchsize: 100,
|
19
|
+
worker_max_lifetime: 3600
|
20
|
+
)
|
21
|
+
|
22
|
+
add_pool Job1,
|
23
|
+
worker_late: 60,
|
24
|
+
pool_min_workers: 1,
|
25
|
+
pool_max_workers: 10
|
26
|
+
|
27
|
+
after_fork do
|
28
|
+
# reset ActiveRecord and other pools
|
29
|
+
puts "Forked!"
|
30
|
+
end
|
data/demo/demo3
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
#!/usr/bin/env ruby bin/hive
|
2
|
+
# -*- encoding: utf-8 -*-
|
3
|
+
|
4
|
+
# This file is evaluated in the context of a Hive::Configuration instance.
|
5
|
+
# DSL attributes include: env, set_env, name, set_name (best to set these via command options)
|
6
|
+
# DSL methods include: chdir, add_path, set_defaults, add_pool, after_fork.
|
7
|
+
|
8
|
+
add_path File.dirname(__FILE__)
|
9
|
+
require "job3"
|
10
|
+
|
11
|
+
# best to pass these on the command line
|
12
|
+
# set_env "development" if ! env
|
13
|
+
# set_name "demo"
|
14
|
+
# chdir "/tmp/demo"
|
15
|
+
|
16
|
+
set_defaults observers: [ :log ] unless env == "test" # No noise when testing please.
|
17
|
+
|
18
|
+
set_defaults(
|
19
|
+
pool_min_workers: 3,
|
20
|
+
worker_late: 10,
|
21
|
+
worker_hung: 100,
|
22
|
+
batchsize: 100,
|
23
|
+
worker_max_lifetime: 3600,
|
24
|
+
observers: [],
|
25
|
+
storage: :redis
|
26
|
+
)
|
27
|
+
|
28
|
+
add_pool Job3,
|
29
|
+
worker_late: 60,
|
30
|
+
pool_min_workers: 10,
|
31
|
+
pool_max_workers: 20
|
32
|
+
|
33
|
+
after_fork do
|
34
|
+
# reset ActiveRecord and other pools
|
35
|
+
puts "Forked!"
|
36
|
+
end
|
data/demo/job1.rb
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require "hive"
|
2
|
+
|
3
|
+
=begin
|
4
|
+
|
5
|
+
Mostly does nothing.
|
6
|
+
Otherwise runs in 1-10 seconds.
|
7
|
+
|
8
|
+
=end
|
9
|
+
|
10
|
+
class Job1
|
11
|
+
|
12
|
+
def initialize( options = {} )
|
13
|
+
end
|
14
|
+
|
15
|
+
include Hive::Log
|
16
|
+
|
17
|
+
def call(context)
|
18
|
+
p = rand
|
19
|
+
|
20
|
+
case
|
21
|
+
when p < 0.7 # 10%
|
22
|
+
log "No action"
|
23
|
+
false
|
24
|
+
else
|
25
|
+
log "Something"
|
26
|
+
sleep(rand(10))
|
27
|
+
true
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
end # Job1
|
data/demo/job2.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
require "hive"
|
2
|
+
|
3
|
+
=begin
|
4
|
+
|
5
|
+
Sometimes exits, hangs, or aborts.
|
6
|
+
Otherwise does something.
|
7
|
+
|
8
|
+
=end
|
9
|
+
|
10
|
+
class Job2
|
11
|
+
|
12
|
+
def initialize( options = {} )
|
13
|
+
end
|
14
|
+
|
15
|
+
include Hive::Log
|
16
|
+
|
17
|
+
def call(context)
|
18
|
+
p = rand
|
19
|
+
|
20
|
+
case
|
21
|
+
when p < 0.1 # 10%
|
22
|
+
log "Going to exit"
|
23
|
+
exit
|
24
|
+
when p < 0.2 # 10%
|
25
|
+
log "Hang"
|
26
|
+
loop do
|
27
|
+
true
|
28
|
+
end
|
29
|
+
when p < 0.3 # 10%
|
30
|
+
log "Going to abort"
|
31
|
+
abort!
|
32
|
+
when p < 0.9 # 60%
|
33
|
+
log "Nothing to do"
|
34
|
+
false
|
35
|
+
else # 10%
|
36
|
+
log "Doing something..."
|
37
|
+
sleep(rand(10))
|
38
|
+
true
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end # Job2
|
data/demo/job3.rb
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
require "hive"
|
2
|
+
require "hive/checker"
|
3
|
+
|
4
|
+
=begin
|
5
|
+
|
6
|
+
Sometimes does something but is slow.
|
7
|
+
|
8
|
+
=end
|
9
|
+
|
10
|
+
class Job3
|
11
|
+
|
12
|
+
attr :redis
|
13
|
+
attr :storage
|
14
|
+
|
15
|
+
def initialize( options = {} )
|
16
|
+
@redis = Redis.connect url: "redis://127.0.0.1:6379/0"
|
17
|
+
@storage = Hive::Redis::Storage.new(redis)
|
18
|
+
end
|
19
|
+
|
20
|
+
include Hive::Log
|
21
|
+
|
22
|
+
def call(context)
|
23
|
+
page = storage.queue_pop( "Next" )
|
24
|
+
activity_count = storage.map_get "Activity", page
|
25
|
+
last_time = storage.map_get( "Last", page )
|
26
|
+
start = Time.now
|
27
|
+
checker = Checker.new activity_count, last_time, start
|
28
|
+
begin
|
29
|
+
return if ! checker.check? # executes ensure block
|
30
|
+
|
31
|
+
log "Processing #{page} with activity #{checker.activity_count}; estimated delay #{checker.estimated_delay} sec"
|
32
|
+
|
33
|
+
sleep(checker.estimated_delay)
|
34
|
+
# checker.checked(rand)
|
35
|
+
|
36
|
+
log "Processed #{page}; updated activity count to #{checker.activity_count}; estimated next time #{checker.next_time}"
|
37
|
+
|
38
|
+
storage.map_set "Last", page, start.to_i
|
39
|
+
ensure
|
40
|
+
storage.queue_add( "Next", page, checker.next_time )
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end # Job3
|
data/demo/populate.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
File.expand_path(File.dirname(__FILE__)+"/../lib").tap { |d| $: << d unless $:.member?(d) }
|
2
|
+
require "hive"
|
3
|
+
require "hive/squiggly"
|
4
|
+
require "redis"
|
5
|
+
|
6
|
+
redis = Redis.connect url: "redis://127.0.0.1:6379/0"
|
7
|
+
storage = Hive::Redis::Storage.new(redis)
|
8
|
+
|
9
|
+
storage.del "Names"
|
10
|
+
storage.del "Activity"
|
11
|
+
storage.del "Weight"
|
12
|
+
storage.del "Next"
|
13
|
+
|
14
|
+
(1..1000).each do |i|
|
15
|
+
name = Squiggly.subject
|
16
|
+
activity = rand(1000)
|
17
|
+
|
18
|
+
storage.map_set "Names", "Page-#{i}", name
|
19
|
+
storage.map_set "Activity", "Page-#{i}", activity
|
20
|
+
|
21
|
+
storage.queue_add( "Next", "Page-#{i}", 0 )
|
22
|
+
end
|
data/hive.gemspec
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
File.expand_path("../lib", __FILE__).tap { |p| $:.push(p) unless $:.member?(p) }
|
3
|
+
|
4
|
+
require "hive/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = "mlanett-hive"
|
8
|
+
s.version = Hive::VERSION
|
9
|
+
s.authors = ["Mark Lanett"]
|
10
|
+
s.email = ["mark.lanett@gmail.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{Manage a collection of worker processes}
|
13
|
+
|
14
|
+
s.files = `git ls-files`.split("\n")
|
15
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
|
+
s.require_paths = ["lib"]
|
18
|
+
|
19
|
+
s.add_dependency "redis"
|
20
|
+
s.add_dependency "redis-namespace"
|
21
|
+
end
|
data/lib/hive.rb
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
File.expand_path(File.dirname(__FILE__)).tap { |d| $: << d unless $:.member?(d) }
|
2
|
+
|
3
|
+
require "hive/version"
|
4
|
+
|
5
|
+
module Hive
|
6
|
+
autoload :Configuration, "hive/configuration"
|
7
|
+
autoload :Idler, "hive/idler"
|
8
|
+
autoload :Key, "hive/key"
|
9
|
+
autoload :LifecycleObserver,"hive/lifecycle_observer"
|
10
|
+
autoload :Log, "hive/log"
|
11
|
+
autoload :Messager, "hive/messager"
|
12
|
+
autoload :Monitor, "hive/monitor"
|
13
|
+
autoload :Policy, "hive/policy"
|
14
|
+
autoload :Pool, "hive/pool"
|
15
|
+
autoload :Registry, "hive/registry"
|
16
|
+
autoload :Trace, "hive/trace"
|
17
|
+
autoload :Worker, "hive/worker"
|
18
|
+
end
|
19
|
+
|
20
|
+
module Hive::Mocks
|
21
|
+
autoload :Storage, "hive/mocks/storage"
|
22
|
+
end
|
23
|
+
|
24
|
+
module Hive::Redis
|
25
|
+
autoload :Storage, "hive/redis/storage"
|
26
|
+
end
|
27
|
+
|
28
|
+
module Hive::Utilities
|
29
|
+
autoload :AirbrakeObserver, "hive/utilities/airbrake_observer"
|
30
|
+
autoload :HoptoadObserver, "hive/utilities/hoptoad_observer"
|
31
|
+
autoload :LogObserver, "hive/utilities/log_observer"
|
32
|
+
autoload :NullObserver, "hive/utilities/null_observer"
|
33
|
+
autoload :Observeable, "hive/utilities/observeable"
|
34
|
+
autoload :ObserverBase, "hive/utilities/observer_base"
|
35
|
+
autoload :Process, "hive/utilities/process"
|
36
|
+
autoload :Resolver, "hive/utilities/resolver"
|
37
|
+
autoload :SignalHook, "hive/utilities/signal_hook"
|
38
|
+
autoload :StorageBase, "hive/utilities/storage_base"
|
39
|
+
end
|
40
|
+
|
41
|
+
class Hive::ConfigurationError < Exception
|
42
|
+
end
|
data/lib/hive/checker.rb
ADDED
@@ -0,0 +1,51 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
class Checker
|
4
|
+
|
5
|
+
DAY = 86400
|
6
|
+
|
7
|
+
# @param activity_count is a daily level of activities
|
8
|
+
# @param last_time is the last time we checked
|
9
|
+
def initialize( activity_count, last_time, check_time = Time.now )
|
10
|
+
@activity_count = activity_count.to_i # e.g. 192 actions
|
11
|
+
@last_time = Time.at last_time.to_i # e.g. 1 hour ago
|
12
|
+
@check_time = check_time # e.g. now
|
13
|
+
|
14
|
+
@last_time = [ @check_time - DAY, @last_time ].max # reject huge intervals like since epoch
|
15
|
+
@interval = @check_time - @last_time # e.g. 1 hour
|
16
|
+
|
17
|
+
# estimations
|
18
|
+
variation = 2 * ( rand + rand + rand + rand ) / 4 # ~1.0 more or less
|
19
|
+
day_est = variation * @activity_count # e.g. ~192
|
20
|
+
@estimate = ( day_est * @interval / DAY ).to_i # e.g. 8 (per hour)
|
21
|
+
next_interval = @estimate > 0 ? @interval / @estimate : @interval * 2
|
22
|
+
end
|
23
|
+
|
24
|
+
def estimate
|
25
|
+
@estimate
|
26
|
+
end
|
27
|
+
|
28
|
+
def checked( actual )
|
29
|
+
@activity_count = ( actual * DAY / @interval ).to_i # e.g. 12
|
30
|
+
next_interval = actual > 0 ? @interval / actual : @interval * 2 # e.g. 300 seconds (3600 / 12)
|
31
|
+
@next_time = @check_time + next_interval # e.g. now + 300 seconds
|
32
|
+
end
|
33
|
+
|
34
|
+
# refuse to check too often
|
35
|
+
def check?
|
36
|
+
@last_time < @check_time - 100
|
37
|
+
end
|
38
|
+
|
39
|
+
def activity_count
|
40
|
+
@activity_count
|
41
|
+
end
|
42
|
+
|
43
|
+
def estimated_delay
|
44
|
+
@estimate > 0 ? Math.sqrt(@estimate).to_i : 1
|
45
|
+
end
|
46
|
+
|
47
|
+
def next_time
|
48
|
+
# If not determined yet, calculate a safety time past the Facebook api limit
|
49
|
+
@next_time ? @next_time : @check_time + 1000
|
50
|
+
end
|
51
|
+
end
|