loops 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.gitignore +8 -0
  2. data/LICENSE +21 -0
  3. data/README.rdoc +238 -0
  4. data/Rakefile +48 -0
  5. data/VERSION.yml +5 -0
  6. data/bin/loops +16 -0
  7. data/bin/loops-memory-stats +259 -0
  8. data/generators/loops/loops_generator.rb +28 -0
  9. data/generators/loops/templates/app/loops/APP_README +1 -0
  10. data/generators/loops/templates/app/loops/queue_loop.rb +8 -0
  11. data/generators/loops/templates/app/loops/simple_loop.rb +12 -0
  12. data/generators/loops/templates/config/loops.yml +34 -0
  13. data/generators/loops/templates/script/loops +20 -0
  14. data/init.rb +1 -0
  15. data/lib/loops.rb +167 -0
  16. data/lib/loops/autoload.rb +20 -0
  17. data/lib/loops/base.rb +148 -0
  18. data/lib/loops/cli.rb +35 -0
  19. data/lib/loops/cli/commands.rb +124 -0
  20. data/lib/loops/cli/options.rb +273 -0
  21. data/lib/loops/command.rb +36 -0
  22. data/lib/loops/commands/debug_command.rb +8 -0
  23. data/lib/loops/commands/list_command.rb +11 -0
  24. data/lib/loops/commands/start_command.rb +24 -0
  25. data/lib/loops/commands/stats_command.rb +5 -0
  26. data/lib/loops/commands/stop_command.rb +18 -0
  27. data/lib/loops/daemonize.rb +68 -0
  28. data/lib/loops/engine.rb +207 -0
  29. data/lib/loops/errors.rb +6 -0
  30. data/lib/loops/logger.rb +212 -0
  31. data/lib/loops/process_manager.rb +114 -0
  32. data/lib/loops/queue.rb +78 -0
  33. data/lib/loops/version.rb +31 -0
  34. data/lib/loops/worker.rb +101 -0
  35. data/lib/loops/worker_pool.rb +55 -0
  36. data/loops.gemspec +98 -0
  37. data/spec/loop_lock_spec.rb +61 -0
  38. data/spec/loops/base_spec.rb +92 -0
  39. data/spec/loops/cli_spec.rb +156 -0
  40. data/spec/loops_spec.rb +20 -0
  41. data/spec/rails/another_loop.rb +4 -0
  42. data/spec/rails/app/loops/complex_loop.rb +12 -0
  43. data/spec/rails/app/loops/simple_loop.rb +6 -0
  44. data/spec/rails/config.yml +6 -0
  45. data/spec/rails/config/boot.rb +1 -0
  46. data/spec/rails/config/environment.rb +5 -0
  47. data/spec/rails/config/loops.yml +13 -0
  48. data/spec/spec_helper.rb +110 -0
  49. metadata +121 -0
@@ -0,0 +1,8 @@
1
+ class Loops::Commands::DebugCommand < Loops::Command
2
+ def execute
3
+ Loops.logger.write_to_console = true
4
+ puts "Starting one loop in debug mode: #{options[:args].first}"
5
+ engine.debug_loop!(options[:args].first)
6
+ exit(0)
7
+ end
8
+ end
@@ -0,0 +1,11 @@
1
+ class Loops::Commands::ListCommand < Loops::Command
2
+ def execute
3
+ puts 'Available loops:'
4
+ engine.loops_config.each do |name, config|
5
+ puts " Loop: #{name}" + (config['disabled'] ? ' (disabled)' : '')
6
+ config.each do |k, v|
7
+ puts " - #{k}: #{v}"
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,24 @@
1
+ class Loops::Commands::StartCommand < Loops::Command
2
+ def execute
3
+ # Pid file check
4
+ if Loops::Daemonize.check_pid(Loops.pid_file)
5
+ puts "Can't start, another process exists!"
6
+ exit(1)
7
+ end
8
+
9
+ # Daemonization
10
+ if options[:daemonize]
11
+ app_name = "loops monitor: #{options[:args].join(' ') rescue 'all'}\0"
12
+ Loops::Daemonize.daemonize(app_name)
13
+ end
14
+
15
+ # Pid file creation
16
+ Loops::Daemonize.create_pid(Loops.pid_file)
17
+
18
+ # Workers processing
19
+ engine.start_loops!(options[:args])
20
+
21
+ # Workers exited, cleaning up
22
+ File.delete(Loops.pid_file) rescue nil
23
+ end
24
+ end
@@ -0,0 +1,5 @@
1
+ class Loops::Commands::StatsCommand < Loops::Command
2
+ def execute
3
+ system File.join(Loops::LIB_ROOT, '../bin/loops-memory-stats')
4
+ end
5
+ end
@@ -0,0 +1,18 @@
1
+ class Loops::Commands::StopCommand < Loops::Command
2
+ def execute
3
+ STDOUT.sync = true
4
+ raise "No pid file or a stale pid file!" unless Loops::Daemonize.check_pid(Loops.pid_file)
5
+ pid = Loops::Daemonize.read_pid(Loops.pid_file)
6
+ print "Killing the process: #{pid}: "
7
+
8
+ loop do
9
+ Process.kill('SIGTERM', pid)
10
+ sleep(1)
11
+ break unless Loops::Daemonize.check_pid(Loops.pid_file)
12
+ print(".")
13
+ end
14
+
15
+ puts " Done!"
16
+ exit(0)
17
+ end
18
+ end
@@ -0,0 +1,68 @@
1
+ module Loops
2
+ module Daemonize
3
+ def self.read_pid(pid_file)
4
+ File.open(pid_file) do |f|
5
+ f.gets.to_i
6
+ end
7
+ rescue Errno::ENOENT
8
+ 0
9
+ end
10
+
11
+ def self.check_pid(pid_file)
12
+ pid = read_pid(pid_file)
13
+ return false if pid.zero?
14
+ if defined?(::JRuby)
15
+ system "kill -0 #{pid} &> /dev/null"
16
+ return $? == 0
17
+ else
18
+ Process.kill(0, pid)
19
+ end
20
+ true
21
+ rescue Errno::ESRCH, Errno::ECHILD, Errno::EPERM
22
+ false
23
+ end
24
+
25
+ def self.create_pid(pid_file)
26
+ if File.exist?(pid_file)
27
+ puts "Pid file #{pid_file} exists! Checking the process..."
28
+ if check_pid(pid_file)
29
+ puts "Can't create new pid file because another process is runnig!"
30
+ return false
31
+ end
32
+ puts "Stale pid file! Removing..."
33
+ File.delete(pid_file)
34
+ end
35
+
36
+ File.open(pid_file, 'w') do |f|
37
+ f.puts(Process.pid)
38
+ end
39
+
40
+ return true
41
+ end
42
+
43
+ def self.daemonize(app_name)
44
+ if defined?(::JRuby)
45
+ puts "WARNING: daemonize method is not implemented for JRuby (yet), please consider using nohup."
46
+ return
47
+ end
48
+
49
+ fork && exit # Fork and exit from the parent
50
+
51
+ # Detach from the controlling terminal
52
+ unless sess_id = Process.setsid
53
+ raise Daemons.RuntimeException.new('cannot detach from controlling terminal')
54
+ end
55
+
56
+ # Prevent the possibility of acquiring a controlling terminal
57
+ trap 'SIGHUP', 'IGNORE'
58
+ exit if pid = fork
59
+
60
+ $0 = app_name if app_name
61
+
62
+ Dir.chdir(Loops.root) # Make sure we're in the working directory
63
+ File.umask(0000) # Insure sensible umask
64
+
65
+ return sess_id
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,207 @@
1
+ class Loops::Engine
2
+ attr_reader :config
3
+
4
+ attr_reader :loops_config
5
+
6
+ attr_reader :global_config
7
+
8
+ def initialize
9
+ load_config
10
+ end
11
+
12
+ def load_config
13
+ # load and parse with erb
14
+ raw_config = File.read(Loops.config_file)
15
+ erb_config = ERB.new(raw_config).result
16
+
17
+ @config = YAML.load(erb_config)
18
+ @loops_config = @config['loops']
19
+ @global_config = {
20
+ 'poll_period' => 1,
21
+ 'workers_engine' => 'fork'
22
+ }.merge(@config['global'])
23
+
24
+ Loops.logger.default_logfile = @global_config['logger'] || $stdout
25
+ Loops.logger.colorful_logs = @global_config['colorful_logs'] || @global_config['colourful_logs']
26
+ end
27
+
28
+ def start_loops!(loops_to_start = [])
29
+ @running_loops = []
30
+ @pm = Loops::ProcessManager.new(global_config, Loops.logger)
31
+
32
+ # Start all loops
33
+ loops_config.each do |name, config|
34
+ next if config['disabled']
35
+ next unless loops_to_start.empty? || loops_to_start.member?(name)
36
+
37
+ klass = load_loop_class(name, config)
38
+ next unless klass
39
+
40
+ start_loop(name, klass, config)
41
+ @running_loops << name
42
+ end
43
+
44
+ # Do not continue if there is nothing to run
45
+ if @running_loops.empty?
46
+ puts 'WARNING: No loops to run! Exiting...'
47
+ return
48
+ end
49
+
50
+ # Start monitoring loop
51
+ setup_signals
52
+ @pm.monitor_workers
53
+
54
+ info 'Loops are stopped now!'
55
+ end
56
+
57
+ def debug_loop!(loop_name)
58
+ @pm = Loops::ProcessManager.new(global_config, Loops.logger)
59
+ loop_config = loops_config[loop_name] || {}
60
+
61
+ # Adjust loop config values before starting it in debug mode
62
+ loop_config['workers_number'] = 1
63
+ loop_config['debug_loop'] = true
64
+
65
+ # Load loop class
66
+ unless klass = load_loop_class(loop_name, loop_config)
67
+ puts "Can't load loop class!"
68
+ return false
69
+ end
70
+
71
+ # Start the loop
72
+ start_loop(loop_name, klass, loop_config)
73
+ end
74
+
75
+ private
76
+
77
+ # Proxy logger calls to the default loops logger
78
+ [ :debug, :error, :fatal, :info, :warn ].each do |meth_name|
79
+ class_eval <<-EVAL, __FILE__, __LINE__
80
+ def #{meth_name}(message)
81
+ Loops.logger.#{meth_name} "loops[RUNNER/\#{Process.pid}]: \#{message}"
82
+ end
83
+ EVAL
84
+ end
85
+
86
+ def load_loop_class(name, config)
87
+ loop_name = config['loop_name'] || name
88
+
89
+ klass_files = [Loops.loops_root + "#{loop_name}_loop.rb", "#{loop_name}_loop"]
90
+ begin
91
+ klass_file = klass_files.shift
92
+ debug "Loading class file: #{klass_file}"
93
+ require(klass_file)
94
+ rescue LoadError
95
+ retry unless klass_files.empty?
96
+ error "Can't load the class file: #{klass_file}. Worker #{name} won't be started!"
97
+ return false
98
+ end
99
+
100
+ klass_name = "#{loop_name}_loop".capitalize.gsub(/_(.)/) { $1.upcase }
101
+ klass = Object.const_get(klass_name) rescue nil
102
+
103
+ unless klass
104
+ error "Can't find class: #{klass_name}. Worker #{name} won't be started!"
105
+ return false
106
+ end
107
+
108
+ begin
109
+ klass.check_dependencies
110
+ rescue Exception => e
111
+ error "Loop #{name} dependencies check failed: #{e} at #{e.backtrace.first}"
112
+ return false
113
+ end
114
+
115
+ return klass
116
+ end
117
+
118
+ def start_loop(name, klass, config)
119
+ info "Starting loop: #{name}"
120
+ info " - config: #{config.inspect}"
121
+
122
+ loop_proc = Proc.new do
123
+ the_logger = begin
124
+ if Loops.logger.is_a?(Loops::Logger) && @global_config['workers_engine'] == 'fork'
125
+ # this is happening right after the fork, therefore no need for teardown at the end of the proc
126
+ Loops.logger.logfile = config['logger']
127
+ Loops.logger
128
+ else
129
+ # for backwards compatibility and handling threading engine
130
+ create_logger(name, config)
131
+ end
132
+ end
133
+
134
+ # Set logger level
135
+ if String === config['log_level']
136
+ level = Logger::Severity.const_get(config['log_level'].upcase) rescue nil
137
+ the_logger.level = level if level
138
+ elsif Integer === config['log_level']
139
+ the_logger.level = config['log_level']
140
+ end
141
+
142
+ # Colorize logging?
143
+ if the_logger.respond_to?(:colorful_logs=) && (config.has_key?('colorful_logs') || config.has_key?('colourful_logs'))
144
+ the_logger.colorful_logs = config['colorful_logs'] || config['colourful_logs']
145
+ end
146
+
147
+ debug "Instantiating class: #{klass}"
148
+ the_loop = klass.new(@pm, name, config)
149
+
150
+ debug "Starting the loop #{name}!"
151
+ fix_ar_after_fork
152
+ # reseed the random number generator in case Loops calls
153
+ # srand or rand prior to forking
154
+ srand
155
+ the_loop.run
156
+ end
157
+
158
+ # If the loop is in debug mode, no need to use all kinds of
159
+ # process managers here
160
+ if config['debug_loop']
161
+ loop_proc.call
162
+ else
163
+ @pm.start_workers(name, config['workers_number'] || 1) { loop_proc.call }
164
+ end
165
+ end
166
+
167
+ def create_logger(loop_name, config)
168
+ config['logger'] ||= 'default'
169
+
170
+ return Loops.default_logger if config['logger'] == 'default'
171
+ Loops::Logger.new(config['logger'])
172
+
173
+ rescue Exception => e
174
+ message = "Can't create a logger for the #{loop_name} loop! Will log to the default logger!"
175
+ puts "ERROR: #{message}"
176
+
177
+ message << "\nException: #{e} at #{e.backtrace.first}"
178
+ error(message)
179
+
180
+ return Loops.default_logger
181
+ end
182
+
183
+ def setup_signals
184
+ trap('TERM') {
185
+ warn "Received a TERM signal... stopping..."
186
+ @pm.stop_workers!
187
+ }
188
+
189
+ trap('INT') {
190
+ warn "Received an INT signal... stopping..."
191
+ @pm.stop_workers!
192
+ }
193
+
194
+ trap('EXIT') {
195
+ warn "Received a EXIT 'signal'... stopping..."
196
+ @pm.stop_workers!
197
+ }
198
+ end
199
+
200
+ def fix_ar_after_fork
201
+ if Object.const_defined?('ActiveRecord')
202
+ ActiveRecord::Base.allow_concurrency = true
203
+ ActiveRecord::Base.clear_active_connections!
204
+ ActiveRecord::Base.verify_active_connections!
205
+ end
206
+ end
207
+ end
@@ -0,0 +1,6 @@
1
+ module Loops::Errors
2
+ Error = Class.new(RuntimeError)
3
+
4
+ InvalidFrameworkError = Class.new(Error)
5
+ InvalidCommandError = Class.new(Error)
6
+ end
@@ -0,0 +1,212 @@
1
+ require 'logger'
2
+ require 'delegate'
3
+ require 'fileutils'
4
+
5
+ class Loops::Logger < ::Delegator
6
+ # @return [Boolean]
7
+ # A value indicating whether all logging output should be
8
+ # also duplicated to the console.
9
+ attr_reader :write_to_console
10
+
11
+ # @return [Boolean]
12
+ # A value inidicating whether critical errors should be highlighted
13
+ # with ANSI colors in the log.
14
+ attr_reader :colorful_logs
15
+
16
+ # Initializes a new instance of the {Logger} class.
17
+ #
18
+ # @param [String, IO] logfile
19
+ # The log device. This is a filename (String), <tt>'stdout'</tt> or
20
+ # <tt>'stderr'</tt> (String), <tt>'default'</tt> for default framework's
21
+ # log file, or +IO+ object (typically +STDOUT+, +STDERR+,
22
+ # or an open file).
23
+ # @param [Integer] level
24
+ # Logging level. Constants are defined in +Logger+ namespace: +DEBUG+, +INFO+,
25
+ # +WARN+, +ERROR+, +FATAL+, or +UNKNOWN+.
26
+ # @param [Integer] number_of_files
27
+ # A number of files to keep.
28
+ # @param [Integer] max_file_size
29
+ # A max file size. When file become larger, next one will be created.
30
+ # @param [Boolean] write_to_console
31
+ # When +true+, all logging output will be dumped to the +STDOUT+ also.
32
+ #
33
+ def initialize(logfile = $stdout, level = ::Logger::INFO, number_of_files = 10, max_file_size = 100 * 1024 * 1024,
34
+ write_to_console = false)
35
+ @number_of_files, @level, @max_file_size, @write_to_console =
36
+ number_of_files, level, max_file_size, write_to_console
37
+ self.logfile = logfile
38
+ super(@implementation)
39
+ end
40
+
41
+ # Sets the default log file (see {#logfile=}).
42
+ #
43
+ # @param [String, IO] logfile
44
+ # the log file path or IO.
45
+ # @return [String, IO]
46
+ # the log file path or IO.
47
+ #
48
+ def default_logfile=(logfile)
49
+ @default_logfile = logfile
50
+ self.logfile = logfile
51
+ end
52
+
53
+ # Sets the log file.
54
+ #
55
+ # @param [String, IO] logfile
56
+ # The log device. This is a filename (String), <tt>'stdout'</tt> or
57
+ # <tt>'stderr'</tt> (String), <tt>'default'</tt> for default framework's
58
+ # log file, or +IO+ object (typically +STDOUT+, +STDERR+,
59
+ # or an open file).
60
+ # @return [String, IO]
61
+ # the log device.
62
+ #
63
+ def logfile=(logfile)
64
+ logfile = @default_logfile || $stdout if logfile == 'default'
65
+ coerced_logfile =
66
+ case logfile
67
+ when 'stdout' then $stdout
68
+ when 'stderr' then $stderr
69
+ when IO, StringIO then logfile
70
+ else
71
+ if Loops.root
72
+ logfile =~ /^\// ? logfile : Loops.root.join(logfile).to_s
73
+ else
74
+ logfile
75
+ end
76
+ end
77
+ # Ensure logging directory does exist
78
+ FileUtils.mkdir_p(File.dirname(coerced_logfile)) if String === coerced_logfile
79
+
80
+ # Create a logger implementation.
81
+ @implementation = LoggerImplementation.new(coerced_logfile, @number_of_files, @max_file_size, @write_to_console, @colorful_logs)
82
+ @implementation.level = @level
83
+ logfile
84
+ end
85
+
86
+ # Remember the level at the proxy level.
87
+ #
88
+ # @param [Integer] level
89
+ # Logging severity.
90
+ # @return [Integer]
91
+ # Logging severity.
92
+ #
93
+ def level=(level)
94
+ @level = level
95
+ @implementation.level = @level if @implementation
96
+ level
97
+ end
98
+
99
+ # Sets a value indicating whether to dump all logs to the console.
100
+ #
101
+ # @param [Boolean] value
102
+ # a value indicating whether to dump all logs to the console.
103
+ # @return [Boolean]
104
+ # a value indicating whether to dump all logs to the console.
105
+ #
106
+ def write_to_console=(value)
107
+ @write_to_console = value
108
+ @implementation.write_to_console = value if @implementation
109
+ value
110
+ end
111
+
112
+ # Sets a value indicating whether to highlight with red ANSI color
113
+ # all critical messages.
114
+ #
115
+ # @param [Boolean] value
116
+ # a value indicating whether to highlight critical errors in log.
117
+ # @return [Boolean]
118
+ # a value indicating whether to highlight critical errors in log.
119
+ #
120
+ def colorful_logs=(value)
121
+ @colorful_logs = value
122
+ @implementation.colorful_logs = value if @implementation
123
+ value
124
+ end
125
+
126
+ # @private
127
+ # Send everything else to @implementation.
128
+ def __getobj__
129
+ @implementation or raise "Logger implementation not initialized"
130
+ end
131
+
132
+ # @private
133
+ # Delegator's method_missing ignores the &block argument (!!!?)
134
+ def method_missing(m, *args, &block)
135
+ target = self.__getobj__
136
+ unless target.respond_to?(m)
137
+ super(m, *args, &block)
138
+ else
139
+ target.__send__(m, *args, &block)
140
+ end
141
+ end
142
+
143
+ # @private
144
+ class LoggerImplementation < ::Logger
145
+
146
+ attr_reader :prefix
147
+
148
+ attr_accessor :write_to_console, :colorful_logs
149
+
150
+ class Formatter
151
+
152
+ def initialize(logger)
153
+ @logger = logger
154
+ end
155
+
156
+ def call(severity, time, progname, message)
157
+ if (@logger.prefix || '').empty?
158
+ "#{severity[0..0]} : #{time.strftime('%Y-%m-%d %H:%M:%S')} : #{message || progname}\n"
159
+ else
160
+ "#{severity[0..0]} : #{time.strftime('%Y-%m-%d %H:%M:%S')} : #{@logger.prefix} : #{message || progname}\n"
161
+ end
162
+ end
163
+ end
164
+
165
+ def initialize(log_device, number_of_files = 10, max_file_size = 10 * 1024 * 1024, write_to_console = true, colorful_logs = false)
166
+ super(log_device, number_of_files, max_file_size)
167
+ self.formatter = Formatter.new(self)
168
+ @write_to_console = write_to_console
169
+ @colorful_logs = colorful_logs
170
+ @prefix = nil
171
+ end
172
+
173
+ def add(severity, message = nil, progname = nil, &block)
174
+ begin
175
+ if @colorful_logs
176
+ message = color_errors(severity, message)
177
+ progname = color_errors(severity, progname)
178
+ end
179
+ super(severity, message, progname, &block)
180
+ if @write_to_console && (message || progname)
181
+ puts self.formatter.call(%w(D I W E F A)[severity] || 'A', Time.now, progname, message)
182
+ end
183
+ rescue
184
+ # ignore errors in logging
185
+ end
186
+ end
187
+
188
+ def with_prefix(prefix)
189
+ raise "No block given" unless block_given?
190
+ old_prefix = @prefix
191
+ @prefix = prefix
192
+ begin
193
+ yield
194
+ ensure
195
+ @prefix = old_prefix
196
+ end
197
+ end
198
+
199
+ def color_errors(severity, line)
200
+ if severity < ::Logger::ERROR
201
+ line
202
+ else
203
+ if line && line !~ /\e/
204
+ "\e[31m#{line}\e[0m"
205
+ else
206
+ line
207
+ end
208
+ end
209
+ end
210
+
211
+ end
212
+ end