loops 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.gitignore +8 -0
  2. data/LICENSE +21 -0
  3. data/README.rdoc +238 -0
  4. data/Rakefile +48 -0
  5. data/VERSION.yml +5 -0
  6. data/bin/loops +16 -0
  7. data/bin/loops-memory-stats +259 -0
  8. data/generators/loops/loops_generator.rb +28 -0
  9. data/generators/loops/templates/app/loops/APP_README +1 -0
  10. data/generators/loops/templates/app/loops/queue_loop.rb +8 -0
  11. data/generators/loops/templates/app/loops/simple_loop.rb +12 -0
  12. data/generators/loops/templates/config/loops.yml +34 -0
  13. data/generators/loops/templates/script/loops +20 -0
  14. data/init.rb +1 -0
  15. data/lib/loops.rb +167 -0
  16. data/lib/loops/autoload.rb +20 -0
  17. data/lib/loops/base.rb +148 -0
  18. data/lib/loops/cli.rb +35 -0
  19. data/lib/loops/cli/commands.rb +124 -0
  20. data/lib/loops/cli/options.rb +273 -0
  21. data/lib/loops/command.rb +36 -0
  22. data/lib/loops/commands/debug_command.rb +8 -0
  23. data/lib/loops/commands/list_command.rb +11 -0
  24. data/lib/loops/commands/start_command.rb +24 -0
  25. data/lib/loops/commands/stats_command.rb +5 -0
  26. data/lib/loops/commands/stop_command.rb +18 -0
  27. data/lib/loops/daemonize.rb +68 -0
  28. data/lib/loops/engine.rb +207 -0
  29. data/lib/loops/errors.rb +6 -0
  30. data/lib/loops/logger.rb +212 -0
  31. data/lib/loops/process_manager.rb +114 -0
  32. data/lib/loops/queue.rb +78 -0
  33. data/lib/loops/version.rb +31 -0
  34. data/lib/loops/worker.rb +101 -0
  35. data/lib/loops/worker_pool.rb +55 -0
  36. data/loops.gemspec +98 -0
  37. data/spec/loop_lock_spec.rb +61 -0
  38. data/spec/loops/base_spec.rb +92 -0
  39. data/spec/loops/cli_spec.rb +156 -0
  40. data/spec/loops_spec.rb +20 -0
  41. data/spec/rails/another_loop.rb +4 -0
  42. data/spec/rails/app/loops/complex_loop.rb +12 -0
  43. data/spec/rails/app/loops/simple_loop.rb +6 -0
  44. data/spec/rails/config.yml +6 -0
  45. data/spec/rails/config/boot.rb +1 -0
  46. data/spec/rails/config/environment.rb +5 -0
  47. data/spec/rails/config/loops.yml +13 -0
  48. data/spec/spec_helper.rb +110 -0
  49. metadata +121 -0
@@ -0,0 +1,61 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe LoopLock do
4
+ before :each do
5
+ @lock = { :entity_id => 1, :loop => 'test' }
6
+ LoopLock.reset!
7
+ end
8
+
9
+ describe '.lock' do
10
+ it 'should lock unlocked entities' do
11
+ LoopLock.lock(@lock).should be_true
12
+ end
13
+
14
+ it 'should create a lock record for unlocked entities' do
15
+ expect {
16
+ LoopLock.lock(@lock)
17
+ }.to change { LoopLock.locked?(@lock) }.from(false).to(true)
18
+ end
19
+
20
+ it 'should not lock an entity more than once' do
21
+ LoopLock.lock(@lock).should be_true
22
+ LoopLock.lock(@lock).should be_false
23
+ end
24
+
25
+ it 'should remove stale locks' do
26
+ @lock[:timeout] = -1 # Expired 1 second ago :-)
27
+ LoopLock.lock(@lock).should be_true
28
+ LoopLock.lock(@lock).should be_true
29
+ end
30
+ end
31
+
32
+ describe '.unlock' do
33
+ before :each do
34
+ LoopLock.lock(@lock)
35
+ end
36
+
37
+ it 'should remove lock records for a locked entities' do
38
+ expect {
39
+ LoopLock.unlock(@lock).should be_true
40
+ }.to change { LoopLock.locked?(@lock) }.from(true).to(false)
41
+ end
42
+
43
+ it 'should gracefully handle situations where we unlock a non-locked entities' do
44
+ LoopLock.reset!
45
+ expect {
46
+ LoopLock.unlock(@lock).should be_false
47
+ }.to_not change { LoopLock.locked?(@lock) }
48
+ end
49
+ end
50
+
51
+ describe '.locked?' do
52
+ it 'should return true for a locked entity' do
53
+ LoopLock.lock(@lock)
54
+ LoopLock.locked?(@lock).should be_true
55
+ end
56
+
57
+ it 'should return false for a non-locked entity' do
58
+ LoopLock.locked?(@lock).should be_false
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,92 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops::Base, '#with_lock' do
4
+ before :each do
5
+ @logger = mock('Logger').as_null_object
6
+ @loop = Loops::Base.new(@logger, 'simple', {})
7
+ end
8
+
9
+ context 'when an entity is not locked' do
10
+ it 'should create lock on an item' do
11
+ called = false
12
+ @loop.with_lock(1, 'rspec', 60) do
13
+ called = true
14
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
15
+ end
16
+ called.should be_true
17
+ end
18
+
19
+ it 'should release lock on an item' do
20
+ called = false
21
+ @loop.with_lock(1, 'rspec', 60) do
22
+ called = true
23
+ end
24
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
25
+ called.should be_true
26
+ end
27
+
28
+ it 'should release lock on an item in case of error' do
29
+ called = false
30
+ expect {
31
+ @loop.with_lock(1, 'rspec', 60) do
32
+ called = true
33
+ raise 'ouch'
34
+ end
35
+ }.to raise_error('ouch')
36
+ called.should be_true
37
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
38
+ end
39
+
40
+ it 'should pass the lock timeout' do
41
+ called = false
42
+ @loop.with_lock(1, 'rspec', 0.2) do
43
+ called = true
44
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_false
45
+ sleep(0.2)
46
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1).should be_true
47
+ end
48
+ called.should be_true
49
+ end
50
+
51
+ it 'should release the lock on an item' do
52
+ called = false
53
+ @loop.with_lock(1, 'rspec', 60) do
54
+ called = true
55
+ end
56
+ called.should be_true
57
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_false
58
+ end
59
+
60
+ it 'should yield with entity_id value if block accepts the argument' do
61
+ called = false
62
+ @loop.with_lock(1, 'rspec', 60) do |entity_id|
63
+ called = true
64
+ entity_id.should == 1
65
+ end
66
+ called.should be_true
67
+ end
68
+ end
69
+
70
+ context 'when an entity is already locked' do
71
+ before :each do
72
+ LoopLock.lock(:loop => 'rspec', :entity_id => 1)
73
+ end
74
+
75
+ it 'should should not yield' do
76
+ called = false
77
+ @loop.with_lock(1, 'rspec', 60) do
78
+ called = true
79
+ end
80
+ called.should be_false
81
+ end
82
+
83
+ it 'should should not touch the lock object' do
84
+ called = false
85
+ @loop.with_lock(1, 'rspec', 60) do
86
+ called = true
87
+ end
88
+ LoopLock.locked?(:loop => 'rspec', :entity_id => 1).should be_true
89
+ called.should be_false
90
+ end
91
+ end
92
+ end
@@ -0,0 +1,156 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops::CLI do
4
+ it 'should include Loops::CLI::Options' do
5
+ Loops::CLI.included_modules.should include(Loops::CLI::Options)
6
+ end
7
+
8
+ it 'should include Loops::CLI::Commands' do
9
+ Loops::CLI.included_modules.should include(Loops::CLI::Commands)
10
+ end
11
+
12
+ describe 'with Loops::CLI::Options included' do
13
+ before :each do
14
+ @args = [ 'list', '-f', 'none']
15
+ end
16
+
17
+ context 'when current directory could be detected' do
18
+ before :each do
19
+ Dir.chdir(RAILS_ROOT)
20
+ end
21
+
22
+ it 'should detect root directory' do
23
+ Loops::CLI.parse(@args)
24
+ Loops.root.should == Pathname.new(RAILS_ROOT).realpath
25
+ end
26
+
27
+ it 'should chdir to the root directory' do
28
+ Loops::CLI.parse(@args)
29
+ Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
30
+ end
31
+
32
+ it 'should load config from config/loops.yml by default' do
33
+ cli = Loops::CLI.parse(@args)
34
+ cli.engine.global_config['pid_file'].should == '/var/run/superloops.pid'
35
+ end
36
+
37
+ it 'should load config from file specified' do
38
+ cli = Loops::CLI.parse(@args << '-c' << 'config.yml')
39
+ cli.engine.global_config['pid_file'].should == 'tmp/pids/loops.pid'
40
+ end
41
+
42
+ it 'should initialize use app/loops as a root directory for loops by default' do
43
+ Loops::CLI.parse(@args)
44
+ Loops.loops_root.should == Pathname.new(RAILS_ROOT + '/app/loops').realpath
45
+ end
46
+
47
+ it 'should use specified root directory for loops' do
48
+ Loops::CLI.parse(@args << '-l' << '.')
49
+ Loops.loops_root.should == Pathname.new(RAILS_ROOT).realpath
50
+ end
51
+
52
+ it 'should use pid file from global config section' do
53
+ Loops::CLI.parse(@args)
54
+ Loops.pid_file.should == Pathname.new('/var/run/superloops.pid')
55
+ end
56
+
57
+ it 'should absolutize relative pid file path' do
58
+ Loops::CLI.parse(@args << '-c' << 'config.yml')
59
+ Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'tmp/pids/loops.pid'
60
+ end
61
+
62
+ it 'should accept pid file from arguments' do
63
+ Loops::CLI.parse(@args << '-p' << 'superloop.pid')
64
+ Loops.pid_file.should == Pathname.new(RAILS_ROOT).realpath + 'superloop.pid'
65
+ end
66
+
67
+ it 'should extract command when passed' do
68
+ cli = Loops::CLI.parse(@args << 'list')
69
+ cli.options[:command].should == 'list'
70
+ end
71
+
72
+ it 'should extract command arguments when passed' do
73
+ cli = Loops::CLI.parse(@args << 'arg1' << 'arg2')
74
+ cli.options[:command].should == 'list'
75
+ cli.options[:args].should == %w(arg1 arg2)
76
+ end
77
+
78
+ it 'should remove all unnecessary options' do
79
+ cli = Loops::CLI.parse(@args << '-r' << RAILS_ROOT << '-p' << 'loop.pid' << '-c' << 'config.yml' << '-l' << '.' << '-d')
80
+ cli.options.keys.map(&:to_s).sort.should == %w(command args daemonize).sort
81
+ end
82
+ end
83
+
84
+ context 'when root directory passed in arguments' do
85
+ before :each do
86
+ @args << '-r' << File.dirname(__FILE__) + '/../rails'
87
+ end
88
+
89
+ it 'should detect root directory' do
90
+ Loops::CLI.parse(@args)
91
+ Loops.root.should == Pathname.new(RAILS_ROOT).realpath
92
+ end
93
+
94
+ it 'should chdir to the root directory' do
95
+ Loops::CLI.parse(@args)
96
+ Dir.pwd.should == Pathname.new(RAILS_ROOT).realpath.to_s
97
+ end
98
+ end
99
+
100
+ context 'with Rails framework' do
101
+ before :each do
102
+ @args = [ 'list', '-r', File.dirname(__FILE__) + '/../rails' ]
103
+ Loops::CLI.parse(@args)
104
+ end
105
+
106
+ it 'should load boot file' do
107
+ Object.const_defined?('RAILS_BOOT_LOADED').should be_true
108
+ end
109
+
110
+ it 'should load environment file' do
111
+ Object.const_defined?('RAILS_ENVIRONMENT_LOADED').should be_true
112
+ end
113
+
114
+ it 'should inialize default logger' do
115
+ Loops.default_logger.should == 'rails default logger'
116
+ end
117
+ end
118
+ end
119
+
120
+ describe 'with Loops::CLI::Commands included' do
121
+ before :each do
122
+ @args = [ 'list', '-f', 'none', '-r', RAILS_ROOT]
123
+ @cli = Loops::CLI.parse(@args)
124
+ end
125
+
126
+ describe 'in #find_command_possibilities' do
127
+ it 'should return a list of possible commands' do
128
+ @cli.find_command_possibilities('s').should == %w(start stats stop)
129
+ @cli.find_command_possibilities('sta').should == %w(start stats)
130
+ @cli.find_command_possibilities('star').should == %w(start)
131
+ @cli.find_command_possibilities('l').should == %w(list)
132
+ @cli.find_command_possibilities('o').should == []
133
+ end
134
+ end
135
+
136
+ describe 'in #find_command' do
137
+ it 'should raise InvalidCommandError when command is not found' do
138
+ expect {
139
+ @cli.find_command('o')
140
+ }.to raise_error(Loops::InvalidCommandError)
141
+ end
142
+
143
+ it 'should raise InvalidCommandError when ambiguous command matches found' do
144
+ expect {
145
+ @cli.find_command('s')
146
+ }.to raise_error(Loops::InvalidCommandError)
147
+ end
148
+
149
+ it 'should return an instance of command when everything is ok' do
150
+ expect {
151
+ @cli.find_command('star').should be_a(Loops::Commands::StartCommand)
152
+ }.to_not raise_error(Loops::InvalidCommandError)
153
+ end
154
+ end
155
+ end
156
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec/spec_helper'
2
+
3
+ describe Loops do
4
+ describe '.load_config' do
5
+ before :each do
6
+ Loops.root = RAILS_ROOT
7
+ @engine = Loops::Engine.new
8
+ end
9
+
10
+ it 'should load and parse Loops configuration file' do
11
+ @engine.config.should be_an_instance_of(Hash)
12
+ @engine.global_config.should be_an_instance_of(Hash)
13
+ @engine.loops_config.should be_an_instance_of(Hash)
14
+ end
15
+
16
+ it 'should process ERB in config file' do
17
+ @engine.global_config['loops_root'].should == Pathname.new(RAILS_ROOT).realpath.to_s
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,4 @@
1
+ class AnotherLoop < Loops::Base
2
+ def run
3
+ end
4
+ end
@@ -0,0 +1,12 @@
1
+ class ComplexLoop < Loops::Base
2
+ def run
3
+ with_period_of(1.second) do
4
+ if shutdown?
5
+ info("Shutting down!")
6
+ return # exit the loop
7
+ end
8
+
9
+ info("ping")
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,6 @@
1
+ class SimpleLoop < Loops::Base
2
+ def run
3
+ info "Do not show this in log"
4
+ error "Woohoo! I'm in the loop log"
5
+ end
6
+ end
@@ -0,0 +1,6 @@
1
+ global:
2
+ pid_file: tmp/pids/loops.pid
3
+
4
+ loops:
5
+ another:
6
+ logger: logs/loops/another.log
@@ -0,0 +1 @@
1
+ RAILS_BOOT_LOADED = true
@@ -0,0 +1,5 @@
1
+ RAILS_ENVIRONMENT_LOADED = true
2
+
3
+ require 'ostruct'
4
+ Rails = OpenStruct.new([:logger])
5
+ Rails.logger = 'rails default logger'
@@ -0,0 +1,13 @@
1
+ global:
2
+ pid_file: /var/run/superloops.pid
3
+ loops_root: <%= Loops.root.to_s %>
4
+ colorful_logs: true
5
+ logger: logs/loops.log
6
+
7
+ loops:
8
+ simple:
9
+ logger: logs/loops/simple.log
10
+ log_level: error
11
+
12
+ complex:
13
+ logger: logs/loops/complex.log
@@ -0,0 +1,110 @@
1
+ RAILS_ROOT = File.expand_path(File.join(File.dirname(__FILE__), 'rails'))
2
+
3
+ require File.join(File.dirname(__FILE__), '..', 'lib', 'loops')
4
+
5
+ # Represents a lock object for a specific item.
6
+ #
7
+ # Usually you should use only {lock} and {unlock} methods to get
8
+ # entity locked or unlocked correspondingly. Method {locked?} could
9
+ # be used in test to verify lock status.
10
+ #
11
+ # @example Create a lock on an entity
12
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15)
13
+ #
14
+ # @example Create a lock on an entity with timeout 20 minutes
15
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
16
+ #
17
+ # @example Remove a lock from an entity
18
+ # LoopLock.unlock(:loop => 'upload', :entity_id => 15)
19
+ #
20
+ # @example Verify entity locked
21
+ # LoopLock.locked?(:loop => 'upload', :entity_id => 15)
22
+ #
23
+ class LoopLock
24
+ @@locks = []
25
+
26
+ # Resets all locks
27
+ #
28
+ # Used just for testing purpose.
29
+ #
30
+ def self.reset!
31
+ @@locks = []
32
+ end
33
+
34
+ # Locks an entity in a specified namespace (loop).
35
+ #
36
+ # @param [Hash] params a hash of options.
37
+ # @option params [String, Symbol] :loop
38
+ # a loop to lock an entity in. Required.
39
+ # @option params [Integer] :entity_id
40
+ # ID of the entity to lock.
41
+ # @option params [Integer] :timeout (1.year)
42
+ # a timeout in seconds after which lock object should be expired.
43
+ # @return [Boolean]
44
+ # +true+ when locked successfully,
45
+ # +false+ when entity is already locked.
46
+ #
47
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
48
+ #
49
+ # @example
50
+ # LoopLock.lock(:loop => 'upload', :entity_id => 15, :timeout => 20.minutes)
51
+ #
52
+ def self.lock(params)
53
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
54
+
55
+ # Remove all stale locks for this record
56
+ @@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] && lock[:timeout_at] < Time.now }
57
+
58
+ return false if locked?(params)
59
+
60
+ # Create new lock
61
+ attributes = params.dup
62
+ timeout = attributes.delete(:timeout)
63
+ timeout ||= 3600 * 24
64
+ attributes[:timeout_at] = Time.now + timeout
65
+ @@locks << attributes
66
+ true
67
+ end
68
+
69
+ # Unlocks an entity in a specified namespace (loop).
70
+ #
71
+ # @param [Hash] params a hash of options.
72
+ # @option params [String, Symbol] :loop
73
+ # a loop to lock an entity in. Required.
74
+ # @option params [Integer] :entity_id
75
+ # ID of the entity to lock.
76
+ # @return [Boolean]
77
+ # +true+ when unlocked successfully,
78
+ # +false+ when entity was not locked before.
79
+ #
80
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
81
+ #
82
+ # @example
83
+ # LoopLock.unlock(:loop => 'upload', :entity_id => 15)
84
+ #
85
+ def self.unlock(params)
86
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
87
+ !!@@locks.reject! { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
88
+ end
89
+
90
+ # Checks the state of an entity lock.
91
+ #
92
+ # @param [Hash] params a hash of options.
93
+ # @option params [String, Symbol] :loop
94
+ # a loop to lock an entity in. Required.
95
+ # @option params [Integer] :entity_id
96
+ # ID of the entity to lock.
97
+ # @return [Boolean]
98
+ # +true+ when an entity is locked,
99
+ # +false+ when an entity is not locked.
100
+ #
101
+ # @raise ArgumentError when :entity_id or :loop parameter is missing.
102
+ #
103
+ # @example
104
+ # LoopLock.locked?(:loop => 'upload', :entity_id => 15)
105
+ #
106
+ def self.locked?(params)
107
+ raise ArgumentError, 'Not enough params for a lock' unless params[:entity_id] && params[:loop]
108
+ !!@@locks.index { |lock| lock[:loop] == params[:loop] && lock[:entity_id] == params[:entity_id] }
109
+ end
110
+ end