loops 2.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +8 -0
- data/LICENSE +21 -0
- data/README.rdoc +238 -0
- data/Rakefile +48 -0
- data/VERSION.yml +5 -0
- data/bin/loops +16 -0
- data/bin/loops-memory-stats +259 -0
- data/generators/loops/loops_generator.rb +28 -0
- data/generators/loops/templates/app/loops/APP_README +1 -0
- data/generators/loops/templates/app/loops/queue_loop.rb +8 -0
- data/generators/loops/templates/app/loops/simple_loop.rb +12 -0
- data/generators/loops/templates/config/loops.yml +34 -0
- data/generators/loops/templates/script/loops +20 -0
- data/init.rb +1 -0
- data/lib/loops.rb +167 -0
- data/lib/loops/autoload.rb +20 -0
- data/lib/loops/base.rb +148 -0
- data/lib/loops/cli.rb +35 -0
- data/lib/loops/cli/commands.rb +124 -0
- data/lib/loops/cli/options.rb +273 -0
- data/lib/loops/command.rb +36 -0
- data/lib/loops/commands/debug_command.rb +8 -0
- data/lib/loops/commands/list_command.rb +11 -0
- data/lib/loops/commands/start_command.rb +24 -0
- data/lib/loops/commands/stats_command.rb +5 -0
- data/lib/loops/commands/stop_command.rb +18 -0
- data/lib/loops/daemonize.rb +68 -0
- data/lib/loops/engine.rb +207 -0
- data/lib/loops/errors.rb +6 -0
- data/lib/loops/logger.rb +212 -0
- data/lib/loops/process_manager.rb +114 -0
- data/lib/loops/queue.rb +78 -0
- data/lib/loops/version.rb +31 -0
- data/lib/loops/worker.rb +101 -0
- data/lib/loops/worker_pool.rb +55 -0
- data/loops.gemspec +98 -0
- data/spec/loop_lock_spec.rb +61 -0
- data/spec/loops/base_spec.rb +92 -0
- data/spec/loops/cli_spec.rb +156 -0
- data/spec/loops_spec.rb +20 -0
- data/spec/rails/another_loop.rb +4 -0
- data/spec/rails/app/loops/complex_loop.rb +12 -0
- data/spec/rails/app/loops/simple_loop.rb +6 -0
- data/spec/rails/config.yml +6 -0
- data/spec/rails/config/boot.rb +1 -0
- data/spec/rails/config/environment.rb +5 -0
- data/spec/rails/config/loops.yml +13 -0
- data/spec/spec_helper.rb +110 -0
- metadata +121 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe LoopLock do
|
4
|
+
before :each do
|
5
|
+
@lock = { :entity_id => 1, :loop => 'test' }
|
6
|
+
LoopLock.reset!
|
7
|
+
end
|
8
|
+
|
9
|
+
describe '.lock' do
|
10
|
+
it 'should lock unlocked entities' do
|
11
|
+
LoopLock.lock(@lock).should be_true
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should create a lock record for unlocked entities' do
|
15
|
+
expect {
|
16
|
+
LoopLock.lock(@lock)
|
17
|
+
}.to change { LoopLock.locked?(@lock) }.from(false).to(true)
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should not lock an entity more than once' do
|
21
|
+
LoopLock.lock(@lock).should be_true
|
22
|
+
LoopLock.lock(@lock).should be_false
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should remove stale locks' do
|
26
|
+
@lock[:timeout] = -1 # Expired 1 second ago :-)
|
27
|
+
LoopLock.lock(@lock).should be_true
|
28
|
+
LoopLock.lock(@lock).should be_true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
describe '.unlock' do
|
33
|
+
before :each do
|
34
|
+
LoopLock.lock(@lock)
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should remove lock records for a locked entities' do
|
38
|
+
expect {
|
39
|
+
LoopLock.unlock(@lock).should be_true
|
40
|
+
}.to change { LoopLock.locked?(@lock) }.from(true).to(false)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should gracefully handle situations where we unlock a non-locked entities' do
|
44
|
+
LoopLock.reset!
|
45
|
+
expect {
|
46
|
+
LoopLock.unlock(@lock).should be_false
|
47
|
+
}.to_not change { LoopLock.locked?(@lock) }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe '.locked?' do
|
52
|
+
it 'should return true for a locked entity' do
|
53
|
+
LoopLock.lock(@lock)
|
54
|
+
LoopLock.locked?(@lock).should be_true
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should return false for a non-locked entity' do
|
58
|
+
LoopLock.locked?(@lock).should be_false
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops::Base, '#with_lock' do
|
4
|
+
before :each do
|
5
|
+
@logger = mock('Logger').as_null_object
|
6
|
+
@loop = Loops::Base.new(@logger, 'simple', {})
|
7
|
+
end
|
8
|
+
|
9
|
+
context 'when an entity is not locked' do
|
10
|
+
it 'should create lock on an item' do
|
11
|
+
called = false
|
12
|
+
@loop.with_lock(1, 'rspec', 60) do
|
13
|
+
called = true
|
14
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
|
15
|
+
end
|
16
|
+
called.should be_true
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should release lock on an item' do
|
20
|
+
called = false
|
21
|
+
@loop.with_lock(1, 'rspec', 60) do
|
22
|
+
called = true
|
23
|
+
end
|
24
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
25
|
+
called.should be_true
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should release lock on an item in case of error' do
|
29
|
+
called = false
|
30
|
+
expect {
|
31
|
+
@loop.with_lock(1, 'rspec', 60) do
|
32
|
+
called = true
|
33
|
+
raise 'ouch'
|
34
|
+
end
|
35
|
+
}.to raise_error('ouch')
|
36
|
+
called.should be_true
|
37
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should pass the lock timeout' do
|
41
|
+
called = false
|
42
|
+
@loop.with_lock(1, 'rspec', 0.2) do
|
43
|
+
called = true
|
44
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_false
|
45
|
+
sleep(0.2)
|
46
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_true
|
47
|
+
end
|
48
|
+
called.should be_true
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should release the lock on an item' do
|
52
|
+
called = false
|
53
|
+
@loop.with_lock(1, 'rspec', 60) do
|
54
|
+
called = true
|
55
|
+
end
|
56
|
+
called.should be_true
|
57
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'should yield with entity_id value if block accepts the argument' do
|
61
|
+
called = false
|
62
|
+
@loop.with_lock(1, 'rspec', 60) do |entity_id|
|
63
|
+
called = true
|
64
|
+
entity_id.should == 1
|
65
|
+
end
|
66
|
+
called.should be_true
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'when an entity is already locked' do
|
71
|
+
before :each do
|
72
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1)
|
73
|
+
end
|
74
|
+
|
75
|
+
it 'should should not yield' do
|
76
|
+
called = false
|
77
|
+
@loop.with_lock(1, 'rspec', 60) do
|
78
|
+
called = true
|
79
|
+
end
|
80
|
+
called.should be_false
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'should should not touch the lock object' do
|
84
|
+
called = false
|
85
|
+
@loop.with_lock(1, 'rspec', 60) do
|
86
|
+
called = true
|
87
|
+
end
|
88
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
|
89
|
+
called.should be_false
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops::CLI do
|
4
|
+
it 'should include Loops::CLI::Options' do
|
5
|
+
Loops::CLI.included_modules.should include(Loops::CLI::Options)
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should include Loops::CLI::Commands' do
|
9
|
+
Loops::CLI.included_modules.should include(Loops::CLI::Commands)
|
10
|
+
end
|
11
|
+
|
12
|
+
describe 'with Loops::CLI::Options included' do
|
13
|
+
before :each do
|
14
|
+
@args = [ 'list', '-f', 'none']
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'when current directory could be detected' do
|
18
|
+
before :each do
|
19
|
+
Dir.chdir(RAILS_ROOT)
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should detect root directory' do
|
23
|
+
Loops::CLI.parse(@args)
|
24
|
+
Loops.root.should == Pathname.new(RAILS_ROOT).realpath
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should chdir to the root directory' do
|
28
|
+
Loops::CLI.parse(@args)
|
29
|
+
Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'should load config from config/loops.yml by default' do
|
33
|
+
cli = Loops::CLI.parse(@args)
|
34
|
+
cli.engine.global_config['pid_file'].should == '/var/run/superloops.pid'
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should load config from file specified' do
|
38
|
+
cli = Loops::CLI.parse(@args << '-c' << 'config.yml')
|
39
|
+
cli.engine.global_config['pid_file'].should == 'tmp/pids/loops.pid'
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should initialize use app/loops as a root directory for loops by default' do
|
43
|
+
Loops::CLI.parse(@args)
|
44
|
+
Loops.loops_root.should == Pathname.new(RAILS_ROOT + '/app/loops').realpath
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'should use specified root directory for loops' do
|
48
|
+
Loops::CLI.parse(@args << '-l' << '.')
|
49
|
+
Loops.loops_root.should == Pathname.new(RAILS_ROOT).realpath
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should use pid file from global config section' do
|
53
|
+
Loops::CLI.parse(@args)
|
54
|
+
Loops.pid_file.should == Pathname.new('/var/run/superloops.pid')
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should absolutize relative pid file path' do
|
58
|
+
Loops::CLI.parse(@args << '-c' << 'config.yml')
|
59
|
+
Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'tmp/pids/loops.pid'
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'should accept pid file from arguments' do
|
63
|
+
Loops::CLI.parse(@args << '-p' << 'superloop.pid')
|
64
|
+
Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'superloop.pid'
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should extract command when passed' do
|
68
|
+
cli = Loops::CLI.parse(@args << 'list')
|
69
|
+
cli.options[:command].should == 'list'
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'should extract command arguments when passed' do
|
73
|
+
cli = Loops::CLI.parse(@args << 'arg1' << 'arg2')
|
74
|
+
cli.options[:command].should == 'list'
|
75
|
+
cli.options[:args].should == %w(arg1 arg2)
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'should remove all unnecessary options' do
|
79
|
+
cli = Loops::CLI.parse(@args << '-r' << RAILS_ROOT << '-p' << 'loop.pid' << '-c' << 'config.yml' << '-l' << '.' << '-d')
|
80
|
+
cli.options.keys.map(&:to_s).sort.should == %w(command args daemonize).sort
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
context 'when root directory passed in arguments' do
|
85
|
+
before :each do
|
86
|
+
@args << '-r' << File.dirname(__FILE__) + '/../rails'
|
87
|
+
end
|
88
|
+
|
89
|
+
it 'should detect root directory' do
|
90
|
+
Loops::CLI.parse(@args)
|
91
|
+
Loops.root.should == Pathname.new(RAILS_ROOT).realpath
|
92
|
+
end
|
93
|
+
|
94
|
+
it 'should chdir to the root directory' do
|
95
|
+
Loops::CLI.parse(@args)
|
96
|
+
Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'with Rails framework' do
|
101
|
+
before :each do
|
102
|
+
@args = [ 'list', '-r', File.dirname(__FILE__) + '/../rails' ]
|
103
|
+
Loops::CLI.parse(@args)
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should load boot file' do
|
107
|
+
Object.const_defined?('RAILS_BOOT_LOADED').should be_true
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'should load environment file' do
|
111
|
+
Object.const_defined?('RAILS_ENVIRONMENT_LOADED').should be_true
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should inialize default logger' do
|
115
|
+
Loops.default_logger.should == 'rails default logger'
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
describe 'with Loops::CLI::Commands included' do
|
121
|
+
before :each do
|
122
|
+
@args = [ 'list', '-f', 'none', '-r', RAILS_ROOT]
|
123
|
+
@cli = Loops::CLI.parse(@args)
|
124
|
+
end
|
125
|
+
|
126
|
+
describe 'in #find_command_possibilities' do
|
127
|
+
it 'should return a list of possible commands' do
|
128
|
+
@cli.find_command_possibilities('s').should == %w(start stats stop)
|
129
|
+
@cli.find_command_possibilities('sta').should == %w(start stats)
|
130
|
+
@cli.find_command_possibilities('star').should == %w(start)
|
131
|
+
@cli.find_command_possibilities('l').should == %w(list)
|
132
|
+
@cli.find_command_possibilities('o').should == []
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe 'in #find_command' do
|
137
|
+
it 'should raise InvalidCommandError when command is not found' do
|
138
|
+
expect {
|
139
|
+
@cli.find_command('o')
|
140
|
+
}.to raise_error(Loops::InvalidCommandError)
|
141
|
+
end
|
142
|
+
|
143
|
+
it 'should raise InvalidCommandError when ambiguous command matches found' do
|
144
|
+
expect {
|
145
|
+
@cli.find_command('s')
|
146
|
+
}.to raise_error(Loops::InvalidCommandError)
|
147
|
+
end
|
148
|
+
|
149
|
+
it 'should return an instance of command when everything is ok' do
|
150
|
+
expect {
|
151
|
+
@cli.find_command('star').should be_a(Loops::Commands::StartCommand)
|
152
|
+
}.to_not raise_error(Loops::InvalidCommandError)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
data/spec/loops_spec.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops do
|
4
|
+
describe '.load_config' do
|
5
|
+
before :each do
|
6
|
+
Loops.root = RAILS_ROOT
|
7
|
+
@engine = Loops::Engine.new
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'should load and parse Loops configuration file' do
|
11
|
+
@engine.config.should be_an_instance_of(Hash)
|
12
|
+
@engine.global_config.should be_an_instance_of(Hash)
|
13
|
+
@engine.loops_config.should be_an_instance_of(Hash)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should process ERB in config file' do
|
17
|
+
@engine.global_config['loops_root'].should == Pathname.new(RAILS_ROOT).realpath.to_s
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
RAILS_BOOT_LOADED = true
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
RAILS_ROOT = File.expand_path(File.join(File.dirname(__FILE__), 'rails'))
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'loops')
|
4
|
+
|
5
|
+
# Represents a lock object for a specific item.
|
6
|
+
#
|
7
|
+
# Usually you should use only {lock} and {unlock} methods to get
|
8
|
+
# entity locked or unlocked correspondingly. Method {locked?} could
|
9
|
+
# be used in test to verify lock status.
|
10
|
+
#
|
11
|
+
# @example Create a lock on an entity
|
12
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15)
|
13
|
+
#
|
14
|
+
# @example Create a lock on an entity with timeout 20 minutes
|
15
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
|
16
|
+
#
|
17
|
+
# @example Remove a lock from an entity
|
18
|
+
# LoopLock.unlock(:loop => 'upload', :entity_id => 15)
|
19
|
+
#
|
20
|
+
# @example Verify entity locked
|
21
|
+
# LoopLock.locked?(:loop => 'upload', :entity_id => 15)
|
22
|
+
#
|
23
|
+
class LoopLock
|
24
|
+
@@locks = []
|
25
|
+
|
26
|
+
# Resets all locks
|
27
|
+
#
|
28
|
+
# Used just for testing purpose.
|
29
|
+
#
|
30
|
+
def self.reset!
|
31
|
+
@@locks = []
|
32
|
+
end
|
33
|
+
|
34
|
+
# Locks an entity in a specified namespace (loop).
|
35
|
+
#
|
36
|
+
# @param [Hash] params a hash of options.
|
37
|
+
# @option params [String, Symbol] :loop
|
38
|
+
# a loop to lock an entity in. Required.
|
39
|
+
# @option params [Integer] :entity_id
|
40
|
+
# ID of the entity to lock.
|
41
|
+
# @option params [Integer] :timeout (1.year)
|
42
|
+
# a timeout in seconds after which lock object should be expired.
|
43
|
+
# @return [Boolean]
|
44
|
+
# +true+ when locked successfully,
|
45
|
+
# +false+ when entity is already locked.
|
46
|
+
#
|
47
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
48
|
+
#
|
49
|
+
# @example
|
50
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
|
51
|
+
#
|
52
|
+
def self.lock(params)
|
53
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
54
|
+
|
55
|
+
# Remove all stale locks for this record
|
56
|
+
@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] && lock[:timeout_at] < Time.now }
|
57
|
+
|
58
|
+
return false if locked?(params)
|
59
|
+
|
60
|
+
# Create new lock
|
61
|
+
attributes = params.dup
|
62
|
+
timeout = attributes.delete(:timeout)
|
63
|
+
timeout ||= 3600 * 24
|
64
|
+
attributes[:timeout_at] = Time.now + timeout
|
65
|
+
@@locks << attributes
|
66
|
+
true
|
67
|
+
end
|
68
|
+
|
69
|
+
# Unlocks an entity in a specified namespace (loop).
|
70
|
+
#
|
71
|
+
# @param [Hash] params a hash of options.
|
72
|
+
# @option params [String, Symbol] :loop
|
73
|
+
# a loop to lock an entity in. Required.
|
74
|
+
# @option params [Integer] :entity_id
|
75
|
+
# ID of the entity to lock.
|
76
|
+
# @return [Boolean]
|
77
|
+
# +true+ when unlocked successfully,
|
78
|
+
# +false+ when entity was not locked before.
|
79
|
+
#
|
80
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
81
|
+
#
|
82
|
+
# @example
|
83
|
+
# LoopLock.unlock(:loop => 'upload', :entity_id => 15)
|
84
|
+
#
|
85
|
+
def self.unlock(params)
|
86
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
87
|
+
!!@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
|
88
|
+
end
|
89
|
+
|
90
|
+
# Checks the state of an entity lock.
|
91
|
+
#
|
92
|
+
# @param [Hash] params a hash of options.
|
93
|
+
# @option params [String, Symbol] :loop
|
94
|
+
# a loop to lock an entity in. Required.
|
95
|
+
# @option params [Integer] :entity_id
|
96
|
+
# ID of the entity to lock.
|
97
|
+
# @return [Boolean]
|
98
|
+
# +true+ when an entity is locked,
|
99
|
+
# +false+ when an entity is not locked.
|
100
|
+
#
|
101
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
102
|
+
#
|
103
|
+
# @example
|
104
|
+
# LoopLock.locked?(:loop => 'upload', :entity_id => 15)
|
105
|
+
#
|
106
|
+
def self.locked?(params)
|
107
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
108
|
+
!!@@locks.index { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
|
109
|
+
end
|
110
|
+
end
|