loops 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +8 -0
- data/LICENSE +21 -0
- data/README.rdoc +238 -0
- data/Rakefile +48 -0
- data/VERSION.yml +5 -0
- data/bin/loops +16 -0
- data/bin/loops-memory-stats +259 -0
- data/generators/loops/loops_generator.rb +28 -0
- data/generators/loops/templates/app/loops/APP_README +1 -0
- data/generators/loops/templates/app/loops/queue_loop.rb +8 -0
- data/generators/loops/templates/app/loops/simple_loop.rb +12 -0
- data/generators/loops/templates/config/loops.yml +34 -0
- data/generators/loops/templates/script/loops +20 -0
- data/init.rb +1 -0
- data/lib/loops.rb +167 -0
- data/lib/loops/autoload.rb +20 -0
- data/lib/loops/base.rb +148 -0
- data/lib/loops/cli.rb +35 -0
- data/lib/loops/cli/commands.rb +124 -0
- data/lib/loops/cli/options.rb +273 -0
- data/lib/loops/command.rb +36 -0
- data/lib/loops/commands/debug_command.rb +8 -0
- data/lib/loops/commands/list_command.rb +11 -0
- data/lib/loops/commands/start_command.rb +24 -0
- data/lib/loops/commands/stats_command.rb +5 -0
- data/lib/loops/commands/stop_command.rb +18 -0
- data/lib/loops/daemonize.rb +68 -0
- data/lib/loops/engine.rb +207 -0
- data/lib/loops/errors.rb +6 -0
- data/lib/loops/logger.rb +212 -0
- data/lib/loops/process_manager.rb +114 -0
- data/lib/loops/queue.rb +78 -0
- data/lib/loops/version.rb +31 -0
- data/lib/loops/worker.rb +101 -0
- data/lib/loops/worker_pool.rb +55 -0
- data/loops.gemspec +98 -0
- data/spec/loop_lock_spec.rb +61 -0
- data/spec/loops/base_spec.rb +92 -0
- data/spec/loops/cli_spec.rb +156 -0
- data/spec/loops_spec.rb +20 -0
- data/spec/rails/another_loop.rb +4 -0
- data/spec/rails/app/loops/complex_loop.rb +12 -0
- data/spec/rails/app/loops/simple_loop.rb +6 -0
- data/spec/rails/config.yml +6 -0
- data/spec/rails/config/boot.rb +1 -0
- data/spec/rails/config/environment.rb +5 -0
- data/spec/rails/config/loops.yml +13 -0
- data/spec/spec_helper.rb +110 -0
- metadata +121 -0
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe LoopLock do
|
4
|
+
before :each do
|
5
|
+
@lock = { :entity_id => 1, :loop => 'test' }
|
6
|
+
LoopLock.reset!
|
7
|
+
end
|
8
|
+
|
9
|
+
describe '.lock' do
|
10
|
+
it 'should lock unlocked entities' do
|
11
|
+
LoopLock.lock(@lock).should be_true
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'should create a lock record for unlocked entities' do
|
15
|
+
expect {
|
16
|
+
LoopLock.lock(@lock)
|
17
|
+
}.to change { LoopLock.locked?(@lock) }.from(false).to(true)
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'should not lock an entity more than once' do
|
21
|
+
LoopLock.lock(@lock).should be_true
|
22
|
+
LoopLock.lock(@lock).should be_false
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should remove stale locks' do
|
26
|
+
@lock[:timeout] = -1 # Expired 1 second ago :-)
|
27
|
+
LoopLock.lock(@lock).should be_true
|
28
|
+
LoopLock.lock(@lock).should be_true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
describe '.unlock' do
|
33
|
+
before :each do
|
34
|
+
LoopLock.lock(@lock)
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should remove lock records for a locked entities' do
|
38
|
+
expect {
|
39
|
+
LoopLock.unlock(@lock).should be_true
|
40
|
+
}.to change { LoopLock.locked?(@lock) }.from(true).to(false)
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'should gracefully handle situations where we unlock a non-locked entities' do
|
44
|
+
LoopLock.reset!
|
45
|
+
expect {
|
46
|
+
LoopLock.unlock(@lock).should be_false
|
47
|
+
}.to_not change { LoopLock.locked?(@lock) }
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
describe '.locked?' do
|
52
|
+
it 'should return true for a locked entity' do
|
53
|
+
LoopLock.lock(@lock)
|
54
|
+
LoopLock.locked?(@lock).should be_true
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should return false for a non-locked entity' do
|
58
|
+
LoopLock.locked?(@lock).should be_false
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops::Base, '#with_lock' do
|
4
|
+
before :each do
|
5
|
+
@logger = mock('Logger').as_null_object
|
6
|
+
@loop = Loops::Base.new(@logger, 'simple', {})
|
7
|
+
end
|
8
|
+
|
9
|
+
context 'when an entity is not locked' do
|
10
|
+
it 'should create lock on an item' do
|
11
|
+
called = false
|
12
|
+
@loop.with_lock(1, 'rspec', 60) do
|
13
|
+
called = true
|
14
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
|
15
|
+
end
|
16
|
+
called.should be_true
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should release lock on an item' do
|
20
|
+
called = false
|
21
|
+
@loop.with_lock(1, 'rspec', 60) do
|
22
|
+
called = true
|
23
|
+
end
|
24
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
25
|
+
called.should be_true
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should release lock on an item in case of error' do
|
29
|
+
called = false
|
30
|
+
expect {
|
31
|
+
@loop.with_lock(1, 'rspec', 60) do
|
32
|
+
called = true
|
33
|
+
raise 'ouch'
|
34
|
+
end
|
35
|
+
}.to raise_error('ouch')
|
36
|
+
called.should be_true
|
37
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should pass the lock timeout' do
|
41
|
+
called = false
|
42
|
+
@loop.with_lock(1, 'rspec', 0.2) do
|
43
|
+
called = true
|
44
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_false
|
45
|
+
sleep(0.2)
|
46
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_true
|
47
|
+
end
|
48
|
+
called.should be_true
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'should release the lock on an item' do
|
52
|
+
called = false
|
53
|
+
@loop.with_lock(1, 'rspec', 60) do
|
54
|
+
called = true
|
55
|
+
end
|
56
|
+
called.should be_true
|
57
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
|
58
|
+
end
|
59
|
+
|
60
|
+
it 'should yield with entity_id value if block accepts the argument' do
|
61
|
+
called = false
|
62
|
+
@loop.with_lock(1, 'rspec', 60) do |entity_id|
|
63
|
+
called = true
|
64
|
+
entity_id.should == 1
|
65
|
+
end
|
66
|
+
called.should be_true
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
context 'when an entity is already locked' do
|
71
|
+
before :each do
|
72
|
+
LoopLock.lock(:loop => 'rspec', :entity_id => 1)
|
73
|
+
end
|
74
|
+
|
75
|
+
it 'should should not yield' do
|
76
|
+
called = false
|
77
|
+
@loop.with_lock(1, 'rspec', 60) do
|
78
|
+
called = true
|
79
|
+
end
|
80
|
+
called.should be_false
|
81
|
+
end
|
82
|
+
|
83
|
+
it 'should should not touch the lock object' do
|
84
|
+
called = false
|
85
|
+
@loop.with_lock(1, 'rspec', 60) do
|
86
|
+
called = true
|
87
|
+
end
|
88
|
+
LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
|
89
|
+
called.should be_false
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops::CLI do
|
4
|
+
it 'should include Loops::CLI::Options' do
|
5
|
+
Loops::CLI.included_modules.should include(Loops::CLI::Options)
|
6
|
+
end
|
7
|
+
|
8
|
+
it 'should include Loops::CLI::Commands' do
|
9
|
+
Loops::CLI.included_modules.should include(Loops::CLI::Commands)
|
10
|
+
end
|
11
|
+
|
12
|
+
describe 'with Loops::CLI::Options included' do
|
13
|
+
before :each do
|
14
|
+
@args = [ 'list', '-f', 'none']
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'when current directory could be detected' do
|
18
|
+
before :each do
|
19
|
+
Dir.chdir(RAILS_ROOT)
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should detect root directory' do
|
23
|
+
Loops::CLI.parse(@args)
|
24
|
+
Loops.root.should == Pathname.new(RAILS_ROOT).realpath
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should chdir to the root directory' do
|
28
|
+
Loops::CLI.parse(@args)
|
29
|
+
Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
|
30
|
+
end
|
31
|
+
|
32
|
+
it 'should load config from config/loops.yml by default' do
|
33
|
+
cli = Loops::CLI.parse(@args)
|
34
|
+
cli.engine.global_config['pid_file'].should == '/var/run/superloops.pid'
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should load config from file specified' do
|
38
|
+
cli = Loops::CLI.parse(@args << '-c' << 'config.yml')
|
39
|
+
cli.engine.global_config['pid_file'].should == 'tmp/pids/loops.pid'
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should initialize use app/loops as a root directory for loops by default' do
|
43
|
+
Loops::CLI.parse(@args)
|
44
|
+
Loops.loops_root.should == Pathname.new(RAILS_ROOT + '/app/loops').realpath
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'should use specified root directory for loops' do
|
48
|
+
Loops::CLI.parse(@args << '-l' << '.')
|
49
|
+
Loops.loops_root.should == Pathname.new(RAILS_ROOT).realpath
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should use pid file from global config section' do
|
53
|
+
Loops::CLI.parse(@args)
|
54
|
+
Loops.pid_file.should == Pathname.new('/var/run/superloops.pid')
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should absolutize relative pid file path' do
|
58
|
+
Loops::CLI.parse(@args << '-c' << 'config.yml')
|
59
|
+
Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'tmp/pids/loops.pid'
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'should accept pid file from arguments' do
|
63
|
+
Loops::CLI.parse(@args << '-p' << 'superloop.pid')
|
64
|
+
Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'superloop.pid'
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should extract command when passed' do
|
68
|
+
cli = Loops::CLI.parse(@args << 'list')
|
69
|
+
cli.options[:command].should == 'list'
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'should extract command arguments when passed' do
|
73
|
+
cli = Loops::CLI.parse(@args << 'arg1' << 'arg2')
|
74
|
+
cli.options[:command].should == 'list'
|
75
|
+
cli.options[:args].should == %w(arg1 arg2)
|
76
|
+
end
|
77
|
+
|
78
|
+
it 'should remove all unnecessary options' do
|
79
|
+
cli = Loops::CLI.parse(@args << '-r' << RAILS_ROOT << '-p' << 'loop.pid' << '-c' << 'config.yml' << '-l' << '.' << '-d')
|
80
|
+
cli.options.keys.map(&:to_s).sort.should == %w(command args daemonize).sort
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
context 'when root directory passed in arguments' do
|
85
|
+
before :each do
|
86
|
+
@args << '-r' << File.dirname(__FILE__) + '/../rails'
|
87
|
+
end
|
88
|
+
|
89
|
+
it 'should detect root directory' do
|
90
|
+
Loops::CLI.parse(@args)
|
91
|
+
Loops.root.should == Pathname.new(RAILS_ROOT).realpath
|
92
|
+
end
|
93
|
+
|
94
|
+
it 'should chdir to the root directory' do
|
95
|
+
Loops::CLI.parse(@args)
|
96
|
+
Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
context 'with Rails framework' do
|
101
|
+
before :each do
|
102
|
+
@args = [ 'list', '-r', File.dirname(__FILE__) + '/../rails' ]
|
103
|
+
Loops::CLI.parse(@args)
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should load boot file' do
|
107
|
+
Object.const_defined?('RAILS_BOOT_LOADED').should be_true
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'should load environment file' do
|
111
|
+
Object.const_defined?('RAILS_ENVIRONMENT_LOADED').should be_true
|
112
|
+
end
|
113
|
+
|
114
|
+
it 'should inialize default logger' do
|
115
|
+
Loops.default_logger.should == 'rails default logger'
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
describe 'with Loops::CLI::Commands included' do
|
121
|
+
before :each do
|
122
|
+
@args = [ 'list', '-f', 'none', '-r', RAILS_ROOT]
|
123
|
+
@cli = Loops::CLI.parse(@args)
|
124
|
+
end
|
125
|
+
|
126
|
+
describe 'in #find_command_possibilities' do
|
127
|
+
it 'should return a list of possible commands' do
|
128
|
+
@cli.find_command_possibilities('s').should == %w(start stats stop)
|
129
|
+
@cli.find_command_possibilities('sta').should == %w(start stats)
|
130
|
+
@cli.find_command_possibilities('star').should == %w(start)
|
131
|
+
@cli.find_command_possibilities('l').should == %w(list)
|
132
|
+
@cli.find_command_possibilities('o').should == []
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe 'in #find_command' do
|
137
|
+
it 'should raise InvalidCommandError when command is not found' do
|
138
|
+
expect {
|
139
|
+
@cli.find_command('o')
|
140
|
+
}.to raise_error(Loops::InvalidCommandError)
|
141
|
+
end
|
142
|
+
|
143
|
+
it 'should raise InvalidCommandError when ambiguous command matches found' do
|
144
|
+
expect {
|
145
|
+
@cli.find_command('s')
|
146
|
+
}.to raise_error(Loops::InvalidCommandError)
|
147
|
+
end
|
148
|
+
|
149
|
+
it 'should return an instance of command when everything is ok' do
|
150
|
+
expect {
|
151
|
+
@cli.find_command('star').should be_a(Loops::Commands::StartCommand)
|
152
|
+
}.to_not raise_error(Loops::InvalidCommandError)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
data/spec/loops_spec.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec/spec_helper'
|
2
|
+
|
3
|
+
describe Loops do
|
4
|
+
describe '.load_config' do
|
5
|
+
before :each do
|
6
|
+
Loops.root = RAILS_ROOT
|
7
|
+
@engine = Loops::Engine.new
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'should load and parse Loops configuration file' do
|
11
|
+
@engine.config.should be_an_instance_of(Hash)
|
12
|
+
@engine.global_config.should be_an_instance_of(Hash)
|
13
|
+
@engine.loops_config.should be_an_instance_of(Hash)
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should process ERB in config file' do
|
17
|
+
@engine.global_config['loops_root'].should == Pathname.new(RAILS_ROOT).realpath.to_s
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
RAILS_BOOT_LOADED = true
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
RAILS_ROOT = File.expand_path(File.join(File.dirname(__FILE__), 'rails'))
|
2
|
+
|
3
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'loops')
|
4
|
+
|
5
|
+
# Represents a lock object for a specific item.
|
6
|
+
#
|
7
|
+
# Usually you should use only {lock} and {unlock} methods to get
|
8
|
+
# entity locked or unlocked correspondingly. Method {locked?} could
|
9
|
+
# be used in test to verify lock status.
|
10
|
+
#
|
11
|
+
# @example Create a lock on an entity
|
12
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15)
|
13
|
+
#
|
14
|
+
# @example Create a lock on an entity with timeout 20 minutes
|
15
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
|
16
|
+
#
|
17
|
+
# @example Remove a lock from an entity
|
18
|
+
# LoopLock.unlock(:loop => 'upload', :entity_id => 15)
|
19
|
+
#
|
20
|
+
# @example Verify entity locked
|
21
|
+
# LoopLock.locked?(:loop => 'upload', :entity_id => 15)
|
22
|
+
#
|
23
|
+
class LoopLock
|
24
|
+
@@locks = []
|
25
|
+
|
26
|
+
# Resets all locks
|
27
|
+
#
|
28
|
+
# Used just for testing purpose.
|
29
|
+
#
|
30
|
+
def self.reset!
|
31
|
+
@@locks = []
|
32
|
+
end
|
33
|
+
|
34
|
+
# Locks an entity in a specified namespace (loop).
|
35
|
+
#
|
36
|
+
# @param [Hash] params a hash of options.
|
37
|
+
# @option params [String, Symbol] :loop
|
38
|
+
# a loop to lock an entity in. Required.
|
39
|
+
# @option params [Integer] :entity_id
|
40
|
+
# ID of the entity to lock.
|
41
|
+
# @option params [Integer] :timeout (1.year)
|
42
|
+
# a timeout in seconds after which lock object should be expired.
|
43
|
+
# @return [Boolean]
|
44
|
+
# +true+ when locked successfully,
|
45
|
+
# +false+ when entity is already locked.
|
46
|
+
#
|
47
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
48
|
+
#
|
49
|
+
# @example
|
50
|
+
# LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
|
51
|
+
#
|
52
|
+
def self.lock(params)
|
53
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
54
|
+
|
55
|
+
# Remove all stale locks for this record
|
56
|
+
@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] && lock[:timeout_at] < Time.now }
|
57
|
+
|
58
|
+
return false if locked?(params)
|
59
|
+
|
60
|
+
# Create new lock
|
61
|
+
attributes = params.dup
|
62
|
+
timeout = attributes.delete(:timeout)
|
63
|
+
timeout ||= 3600 * 24
|
64
|
+
attributes[:timeout_at] = Time.now + timeout
|
65
|
+
@@locks << attributes
|
66
|
+
true
|
67
|
+
end
|
68
|
+
|
69
|
+
# Unlocks an entity in a specified namespace (loop).
|
70
|
+
#
|
71
|
+
# @param [Hash] params a hash of options.
|
72
|
+
# @option params [String, Symbol] :loop
|
73
|
+
# a loop to lock an entity in. Required.
|
74
|
+
# @option params [Integer] :entity_id
|
75
|
+
# ID of the entity to lock.
|
76
|
+
# @return [Boolean]
|
77
|
+
# +true+ when unlocked successfully,
|
78
|
+
# +false+ when entity was not locked before.
|
79
|
+
#
|
80
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
81
|
+
#
|
82
|
+
# @example
|
83
|
+
# LoopLock.unlock(:loop => 'upload', :entity_id => 15)
|
84
|
+
#
|
85
|
+
def self.unlock(params)
|
86
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
87
|
+
!!@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
|
88
|
+
end
|
89
|
+
|
90
|
+
# Checks the state of an entity lock.
|
91
|
+
#
|
92
|
+
# @param [Hash] params a hash of options.
|
93
|
+
# @option params [String, Symbol] :loop
|
94
|
+
# a loop to lock an entity in. Required.
|
95
|
+
# @option params [Integer] :entity_id
|
96
|
+
# ID of the entity to lock.
|
97
|
+
# @return [Boolean]
|
98
|
+
# +true+ when an entity is locked,
|
99
|
+
# +false+ when an entity is not locked.
|
100
|
+
#
|
101
|
+
# @raise ArgumentError when :entity_id or :loop parameter is missing.
|
102
|
+
#
|
103
|
+
# @example
|
104
|
+
# LoopLock.locked?(:loop => 'upload', :entity_id => 15)
|
105
|
+
#
|
106
|
+
def self.locked?(params)
|
107
|
+
raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
|
108
|
+
!!@@locks.index { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
|
109
|
+
end
|
110
|
+
end
|