loops 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (49) hide show
  1. data/.gitignore +8 -0
  2. data/LICENSE +21 -0
  3. data/README.rdoc +238 -0
  4. data/Rakefile +48 -0
  5. data/VERSION.yml +5 -0
  6. data/bin/loops +16 -0
  7. data/bin/loops-memory-stats +259 -0
  8. data/generators/loops/loops_generator.rb +28 -0
  9. data/generators/loops/templates/app/loops/APP_README +1 -0
  10. data/generators/loops/templates/app/loops/queue_loop.rb +8 -0
  11. data/generators/loops/templates/app/loops/simple_loop.rb +12 -0
  12. data/generators/loops/templates/config/loops.yml +34 -0
  13. data/generators/loops/templates/script/loops +20 -0
  14. data/init.rb +1 -0
  15. data/lib/loops.rb +167 -0
  16. data/lib/loops/autoload.rb +20 -0
  17. data/lib/loops/base.rb +148 -0
  18. data/lib/loops/cli.rb +35 -0
  19. data/lib/loops/cli/commands.rb +124 -0
  20. data/lib/loops/cli/options.rb +273 -0
  21. data/lib/loops/command.rb +36 -0
  22. data/lib/loops/commands/debug_command.rb +8 -0
  23. data/lib/loops/commands/list_command.rb +11 -0
  24. data/lib/loops/commands/start_command.rb +24 -0
  25. data/lib/loops/commands/stats_command.rb +5 -0
  26. data/lib/loops/commands/stop_command.rb +18 -0
  27. data/lib/loops/daemonize.rb +68 -0
  28. data/lib/loops/engine.rb +207 -0
  29. data/lib/loops/errors.rb +6 -0
  30. data/lib/loops/logger.rb +212 -0
  31. data/lib/loops/process_manager.rb +114 -0
  32. data/lib/loops/queue.rb +78 -0
  33. data/lib/loops/version.rb +31 -0
  34. data/lib/loops/worker.rb +101 -0
  35. data/lib/loops/worker_pool.rb +55 -0
  36. data/loops.gemspec +98 -0
  37. data/spec/loop_lock_spec.rb +61 -0
  38. data/spec/loops/base_spec.rb +92 -0
  39. data/spec/loops/cli_spec.rb +156 -0
  40. data/spec/loops_spec.rb +20 -0
  41. data/spec/rails/another_loop.rb +4 -0
  42. data/spec/rails/app/loops/complex_loop.rb +12 -0
  43. data/spec/rails/app/loops/simple_loop.rb +6 -0
  44. data/spec/rails/config.yml +6 -0
  45. data/spec/rails/config/boot.rb +1 -0
  46. data/spec/rails/config/environment.rb +5 -0
  47. data/spec/rails/config/loops.yml +13 -0
  48. data/spec/spec_helper.rb +110 -0
  49. metadata +121 -0
@@ -0,0 +1,61 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe LoopLock do
4
+ before :each do
5
+ @lock = { :entity_id => 1, :loop => 'test' }
6
+ LoopLock.reset!
7
+ end
8
+
9
+ describe '.lock' do
10
+ it 'should lock unlocked entities' do
11
+ LoopLock.lock(@lock).should be_true
12
+ end
13
+
14
+ it 'should create a lock record for unlocked entities' do
15
+ expect {
16
+ LoopLock.lock(@lock)
17
+ }.to change { LoopLock.locked?(@lock) }.from(false).to(true)
18
+ end
19
+
20
+ it 'should not lock an entity more than once' do
21
+ LoopLock.lock(@lock).should be_true
22
+ LoopLock.lock(@lock).should be_false
23
+ end
24
+
25
+ it 'should remove stale locks' do
26
+ @lock[:timeout] = -1 # Expired 1 second ago :-)
27
+ LoopLock.lock(@lock).should be_true
28
+ LoopLock.lock(@lock).should be_true
29
+ end
30
+ end
31
+
32
+ describe '.unlock' do
33
+ before :each do
34
+ LoopLock.lock(@lock)
35
+ end
36
+
37
+ it 'should remove lock records for a locked entities' do
38
+ expect {
39
+ LoopLock.unlock(@lock).should be_true
40
+ }.to change { LoopLock.locked?(@lock) }.from(true).to(false)
41
+ end
42
+
43
+ it 'should gracefully handle situations where we unlock a non-locked entities' do
44
+ LoopLock.reset!
45
+ expect {
46
+ LoopLock.unlock(@lock).should be_false
47
+ }.to_not change { LoopLock.locked?(@lock) }
48
+ end
49
+ end
50
+
51
+ describe '.locked?' do
52
+ it 'should return true for a locked entity' do
53
+ LoopLock.lock(@lock)
54
+ LoopLock.locked?(@lock).should be_true
55
+ end
56
+
57
+ it 'should return false for a non-locked entity' do
58
+ LoopLock.locked?(@lock).should be_false
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,92 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops::Base, '#with_lock' do
4
+ before :each do
5
+ @logger = mock('Logger').as_null_object
6
+ @loop = Loops::Base.new(@logger, 'simple', {})
7
+ end
8
+
9
+ context 'when an entity is not locked' do
10
+ it 'should create lock on an item' do
11
+ called = false
12
+ @loop.with_lock(1, 'rspec', 60) do
13
+ called = true
14
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
15
+ end
16
+ called.should be_true
17
+ end
18
+
19
+ it 'should release lock on an item' do
20
+ called = false
21
+ @loop.with_lock(1, 'rspec', 60) do
22
+ called = true
23
+ end
24
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
25
+ called.should be_true
26
+ end
27
+
28
+ it 'should release lock on an item in case of error' do
29
+ called = false
30
+ expect {
31
+ @loop.with_lock(1, 'rspec', 60) do
32
+ called = true
33
+ raise 'ouch'
34
+ end
35
+ }.to raise_error('ouch')
36
+ called.should be_true
37
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
38
+ end
39
+
40
+ it 'should pass the lock timeout' do
41
+ called = false
42
+ @loop.with_lock(1, 'rspec', 0.2) do
43
+ called = true
44
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_false
45
+ sleep(0.2)
46
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_true
47
+ end
48
+ called.should be_true
49
+ end
50
+
51
+ it 'should release the lock on an item' do
52
+ called = false
53
+ @loop.with_lock(1, 'rspec', 60) do
54
+ called = true
55
+ end
56
+ called.should be_true
57
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
58
+ end
59
+
60
+ it 'should yield with entity_id value if block accepts the argument' do
61
+ called = false
62
+ @loop.with_lock(1, 'rspec', 60) do |entity_id|
63
+ called = true
64
+ entity_id.should == 1
65
+ end
66
+ called.should be_true
67
+ end
68
+ end
69
+
70
+ context 'when an entity is already locked' do
71
+ before :each do
72
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1)
73
+ end
74
+
75
+ it 'should should not yield' do
76
+ called = false
77
+ @loop.with_lock(1, 'rspec', 60) do
78
+ called = true
79
+ end
80
+ called.should be_false
81
+ end
82
+
83
+ it 'should should not touch the lock object' do
84
+ called = false
85
+ @loop.with_lock(1, 'rspec', 60) do
86
+ called = true
87
+ end
88
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
89
+ called.should be_false
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,156 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops::CLI do
4
+ it 'should include Loops::CLI::Options' do
5
+ Loops::CLI.included_modules.should include(Loops::CLI::Options)
6
+ end
7
+
8
+ it 'should include Loops::CLI::Commands' do
9
+ Loops::CLI.included_modules.should include(Loops::CLI::Commands)
10
+ end
11
+
12
+ describe 'with Loops::CLI::Options included' do
13
+ before :each do
14
+ @args = [ 'list', '-f', 'none']
15
+ end
16
+
17
+ context 'when current directory could be detected' do
18
+ before :each do
19
+ Dir.chdir(RAILS_ROOT)
20
+ end
21
+
22
+ it 'should detect root directory' do
23
+ Loops::CLI.parse(@args)
24
+ Loops.root.should == Pathname.new(RAILS_ROOT).realpath
25
+ end
26
+
27
+ it 'should chdir to the root directory' do
28
+ Loops::CLI.parse(@args)
29
+ Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
30
+ end
31
+
32
+ it 'should load config from config/loops.yml by default' do
33
+ cli = Loops::CLI.parse(@args)
34
+ cli.engine.global_config['pid_file'].should == '/var/run/superloops.pid'
35
+ end
36
+
37
+ it 'should load config from file specified' do
38
+ cli = Loops::CLI.parse(@args << '-c' << 'config.yml')
39
+ cli.engine.global_config['pid_file'].should == 'tmp/pids/loops.pid'
40
+ end
41
+
42
+ it 'should initialize use app/loops as a root directory for loops by default' do
43
+ Loops::CLI.parse(@args)
44
+ Loops.loops_root.should == Pathname.new(RAILS_ROOT + '/app/loops').realpath
45
+ end
46
+
47
+ it 'should use specified root directory for loops' do
48
+ Loops::CLI.parse(@args << '-l' << '.')
49
+ Loops.loops_root.should == Pathname.new(RAILS_ROOT).realpath
50
+ end
51
+
52
+ it 'should use pid file from global config section' do
53
+ Loops::CLI.parse(@args)
54
+ Loops.pid_file.should == Pathname.new('/var/run/superloops.pid')
55
+ end
56
+
57
+ it 'should absolutize relative pid file path' do
58
+ Loops::CLI.parse(@args << '-c' << 'config.yml')
59
+ Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'tmp/pids/loops.pid'
60
+ end
61
+
62
+ it 'should accept pid file from arguments' do
63
+ Loops::CLI.parse(@args << '-p' << 'superloop.pid')
64
+ Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'superloop.pid'
65
+ end
66
+
67
+ it 'should extract command when passed' do
68
+ cli = Loops::CLI.parse(@args << 'list')
69
+ cli.options[:command].should == 'list'
70
+ end
71
+
72
+ it 'should extract command arguments when passed' do
73
+ cli = Loops::CLI.parse(@args << 'arg1' << 'arg2')
74
+ cli.options[:command].should == 'list'
75
+ cli.options[:args].should == %w(arg1 arg2)
76
+ end
77
+
78
+ it 'should remove all unnecessary options' do
79
+ cli = Loops::CLI.parse(@args << '-r' << RAILS_ROOT << '-p' << 'loop.pid' << '-c' << 'config.yml' << '-l' << '.' << '-d')
80
+ cli.options.keys.map(&:to_s).sort.should == %w(command args daemonize).sort
81
+ end
82
+ end
83
+
84
+ context 'when root directory passed in arguments' do
85
+ before :each do
86
+ @args << '-r' << File.dirname(__FILE__) + '/../rails'
87
+ end
88
+
89
+ it 'should detect root directory' do
90
+ Loops::CLI.parse(@args)
91
+ Loops.root.should == Pathname.new(RAILS_ROOT).realpath
92
+ end
93
+
94
+ it 'should chdir to the root directory' do
95
+ Loops::CLI.parse(@args)
96
+ Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
97
+ end
98
+ end
99
+
100
+ context 'with Rails framework' do
101
+ before :each do
102
+ @args = [ 'list', '-r', File.dirname(__FILE__) + '/../rails' ]
103
+ Loops::CLI.parse(@args)
104
+ end
105
+
106
+ it 'should load boot file' do
107
+ Object.const_defined?('RAILS_BOOT_LOADED').should be_true
108
+ end
109
+
110
+ it 'should load environment file' do
111
+ Object.const_defined?('RAILS_ENVIRONMENT_LOADED').should be_true
112
+ end
113
+
114
+ it 'should inialize default logger' do
115
+ Loops.default_logger.should == 'rails default logger'
116
+ end
117
+ end
118
+ end
119
+
120
+ describe 'with Loops::CLI::Commands included' do
121
+ before :each do
122
+ @args = [ 'list', '-f', 'none', '-r', RAILS_ROOT]
123
+ @cli = Loops::CLI.parse(@args)
124
+ end
125
+
126
+ describe 'in #find_command_possibilities' do
127
+ it 'should return a list of possible commands' do
128
+ @cli.find_command_possibilities('s').should == %w(start stats stop)
129
+ @cli.find_command_possibilities('sta').should == %w(start stats)
130
+ @cli.find_command_possibilities('star').should == %w(start)
131
+ @cli.find_command_possibilities('l').should == %w(list)
132
+ @cli.find_command_possibilities('o').should == []
133
+ end
134
+ end
135
+
136
+ describe 'in #find_command' do
137
+ it 'should raise InvalidCommandError when command is not found' do
138
+ expect {
139
+ @cli.find_command('o')
140
+ }.to raise_error(Loops::InvalidCommandError)
141
+ end
142
+
143
+ it 'should raise InvalidCommandError when ambiguous command matches found' do
144
+ expect {
145
+ @cli.find_command('s')
146
+ }.to raise_error(Loops::InvalidCommandError)
147
+ end
148
+
149
+ it 'should return an instance of command when everything is ok' do
150
+ expect {
151
+ @cli.find_command('star').should be_a(Loops::Commands::StartCommand)
152
+ }.to_not raise_error(Loops::InvalidCommandError)
153
+ end
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops do
4
+ describe '.load_config' do
5
+ before :each do
6
+ Loops.root = RAILS_ROOT
7
+ @engine = Loops::Engine.new
8
+ end
9
+
10
+ it 'should load and parse Loops configuration file' do
11
+ @engine.config.should be_an_instance_of(Hash)
12
+ @engine.global_config.should be_an_instance_of(Hash)
13
+ @engine.loops_config.should be_an_instance_of(Hash)
14
+ end
15
+
16
+ it 'should process ERB in config file' do
17
+ @engine.global_config['loops_root'].should == Pathname.new(RAILS_ROOT).realpath.to_s
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,4 @@
1
+ class AnotherLoop < Loops::Base
2
+ def run
3
+ end
4
+ end
@@ -0,0 +1,12 @@
1
+ class ComplexLoop < Loops::Base
2
+ def run
3
+ with_period_of(1.second) do
4
+ if shutdown?
5
+ info("Shutting down!")
6
+ return # exit the loop
7
+ end
8
+
9
+ info("ping")
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,6 @@
1
+ class SimpleLoop < Loops::Base
2
+ def run
3
+ info "Do not show this in log"
4
+ error "Woohoo! I'm in the loop log"
5
+ end
6
+ end
@@ -0,0 +1,6 @@
1
+ global:
2
+ pid_file: tmp/pids/loops.pid
3
+
4
+ loops:
5
+ another:
6
+ logger: logs/loops/another.log
@@ -0,0 +1 @@
1
+ RAILS_BOOT_LOADED = true
@@ -0,0 +1,5 @@
1
+ RAILS_ENVIRONMENT_LOADED = true
2
+
3
+ require 'ostruct'
4
+ Rails = OpenStruct.new([:logger])
5
+ Rails.logger = 'rails default logger'
@@ -0,0 +1,13 @@
1
+ global:
2
+ pid_file: /var/run/superloops.pid
3
+ loops_root: <%= Loops.root.to_s %>
4
+ colorful_logs: true
5
+ logger: logs/loops.log
6
+
7
+ loops:
8
+ simple:
9
+ logger: logs/loops/simple.log
10
+ log_level: error
11
+
12
+ complex:
13
+ logger: logs/loops/complex.log
@@ -0,0 +1,110 @@
1
+ RAILS_ROOT = File.expand_path(File.join(File.dirname(__FILE__), 'rails'))
2
+
3
+ require File.join(File.dirname(__FILE__), '..', 'lib', 'loops')
4
+
5
+ # Represents a lock object for a specific item.
6
+ #
7
+ # Usually you should use only {lock} and {unlock} methods to get
8
+ # entity locked or unlocked correspondingly. Method {locked?} could
9
+ # be used in test to verify lock status.
10
+ #
11
+ # @example Create a lock on an entity
12
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15)
13
+ #
14
+ # @example Create a lock on an entity with timeout 20 minutes
15
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
16
+ #
17
+ # @example Remove a lock from an entity
18
+ # LoopLock.unlock(:loop => 'upload', :entity_id => 15)
19
+ #
20
+ # @example Verify entity locked
21
+ # LoopLock.locked?(:loop => 'upload', :entity_id => 15)
22
+ #
23
+ class LoopLock
24
+ @@locks = []
25
+
26
+ # Resets all locks
27
+ #
28
+ # Used just for testing purpose.
29
+ #
30
+ def self.reset!
31
+ @@locks = []
32
+ end
33
+
34
+ # Locks an entity in a specified namespace (loop).
35
+ #
36
+ # @param [Hash] params a hash of options.
37
+ # @option params [String, Symbol] :loop
38
+ # a loop to lock an entity in. Required.
39
+ # @option params [Integer] :entity_id
40
+ # ID of the entity to lock.
41
+ # @option params [Integer] :timeout (1.year)
42
+ # a timeout in seconds after which lock object should be expired.
43
+ # @return [Boolean]
44
+ # +true+ when locked successfully,
45
+ # +false+ when entity is already locked.
46
+ #
47
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
48
+ #
49
+ # @example
50
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
51
+ #
52
+ def self.lock(params)
53
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
54
+
55
+ # Remove all stale locks for this record
56
+ @@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] && lock[:timeout_at] < Time.now }
57
+
58
+ return false if locked?(params)
59
+
60
+ # Create new lock
61
+ attributes = params.dup
62
+ timeout = attributes.delete(:timeout)
63
+ timeout ||= 3600 * 24
64
+ attributes[:timeout_at] = Time.now + timeout
65
+ @@locks << attributes
66
+ true
67
+ end
68
+
69
+ # Unlocks an entity in a specified namespace (loop).
70
+ #
71
+ # @param [Hash] params a hash of options.
72
+ # @option params [String, Symbol] :loop
73
+ # a loop to lock an entity in. Required.
74
+ # @option params [Integer] :entity_id
75
+ # ID of the entity to lock.
76
+ # @return [Boolean]
77
+ # +true+ when unlocked successfully,
78
+ # +false+ when entity was not locked before.
79
+ #
80
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
81
+ #
82
+ # @example
83
+ # LoopLock.unlock(:loop => 'upload', :entity_id => 15)
84
+ #
85
+ def self.unlock(params)
86
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
87
+ !!@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
88
+ end
89
+
90
+ # Checks the state of an entity lock.
91
+ #
92
+ # @param [Hash] params a hash of options.
93
+ # @option params [String, Symbol] :loop
94
+ # a loop to lock an entity in. Required.
95
+ # @option params [Integer] :entity_id
96
+ # ID of the entity to lock.
97
+ # @return [Boolean]
98
+ # +true+ when an entity is locked,
99
+ # +false+ when an entity is not locked.
100
+ #
101
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
102
+ #
103
+ # @example
104
+ # LoopLock.locked?(:loop => 'upload', :entity_id => 15)
105
+ #
106
+ def self.locked?(params)
107
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
108
+ !!@@locks.index { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
109
+ end
110
+ end