zmq_jobs 0.0.1.alpha2 → 0.0.1.alpha3

Sign up to get free protection for your applications and to get access to all the features.
@@ -20,9 +20,11 @@ module ZmqJobs
20
20
  def initialize(args)
21
21
  raise 'You can not create instance of abstract class' if self.class == Command
22
22
  @daemonize = false
23
+ @debug = false
23
24
  @monitor = false
24
25
  @execute_dir = Dir.pwd
25
- @pid_dir = './tmp'
26
+ @log_dir = './log'
27
+ @pid_dir = './pids'
26
28
  @config_file = './config/zmq_jobs.yml'
27
29
  @options = read_config_file
28
30
 
@@ -57,6 +59,9 @@ module ZmqJobs
57
59
  opts.on('-d', '--daemonize', "Daemonize the #{type} process") do
58
60
  @daemonize = true
59
61
  end
62
+ opts.on('-D', '--debug', "Debug mode") do
63
+ @debug = true
64
+ end
60
65
  opts.on('-c CONFIG', '--config CONFIG', "Use config file, default: #{config_file}") do |config|
61
66
  @config_file = config
62
67
  end
@@ -73,6 +78,7 @@ module ZmqJobs
73
78
  Daemons.run_proc(
74
79
  daemon_name,
75
80
  {
81
+ :dir => @pid_dir,
76
82
  :multiple => false,
77
83
  :dir_mode => :normal,
78
84
  :monitor => monitor,
@@ -80,6 +86,8 @@ module ZmqJobs
80
86
  :ontop => !daemonize
81
87
  }
82
88
  ) do
89
+ config['debug'] = true if @debug
90
+ ZmqJobs.logger = Logger.new(File.expand_path("#{@log_dir}/#{daemon_name}.log", execute_dir)) if daemonize
83
91
  daemon_class(daemon_name).new(config).start
84
92
  end
85
93
  true
@@ -134,10 +142,11 @@ module ZmqJobs
134
142
 
135
143
  protected
136
144
  def preload_worker_class classname
145
+ filename = ActiveSupport::Inflector.underscore(classname)
137
146
  worker_class_dir = File.expand_path(options['workers_dir'], execute_dir)
138
147
  Kernel.require(
139
- "#{worker_class_dir}/#{ActiveSupport::Inflector.underscore(classname)}"
140
- ) unless Kernel.const_defined?(classname)
148
+ "#{worker_class_dir}/#{filename}"
149
+ ) unless Kernel.const_defined?(classname) && !File.exists?("#{worker_class_dir}/#{filename}.rb")
141
150
  end
142
151
 
143
152
  def opts_parser
@@ -145,6 +154,9 @@ module ZmqJobs
145
154
  opts.on('-w', '--workers LIST', 'Workers which have to run.') do |workers|
146
155
  @workers = workers.split(',')
147
156
  end
157
+ # opts.on('-r REQUIRE_FILE', '--require REQUIRE_FILE', "Require this file before start daemon") do |require_file|
158
+ # @require = require_file
159
+ # end
148
160
  end
149
161
  end
150
162
 
@@ -0,0 +1,5 @@
1
+ module ZmqJobs
2
+ class Railties < Rails::Railties
3
+ config.autoload_paths << Rails.root.join('app', 'workers')
4
+ end
5
+ end
@@ -50,3 +50,5 @@ module ZmqJobs
50
50
  end
51
51
 
52
52
  require 'command'
53
+
54
+ require 'rails/railties' if defined?(Rails)
@@ -3,6 +3,9 @@ module ZmqJobs
3
3
  protected
4
4
  def start_device
5
5
  @device = ZMQ::Device.new(ZMQ::FORWARDER, frontend.socket, backend.socket)
6
+ rescue => e
7
+ logger.warn format_exception_message(e)
8
+ raise e
6
9
  end
7
10
 
8
11
  def create_frontend
@@ -61,5 +61,12 @@ module ZmqJobs
61
61
  def logger
62
62
  ::ZmqJobs.logger
63
63
  end
64
+
65
+ def format_exception_message exception
66
+ <<-MESSAGE
67
+ Execution error: #{exception} - #{exception.message}
68
+ #{exception.backtrace[0..10].join("\n ")}
69
+ MESSAGE
70
+ end
64
71
  end
65
72
  end
@@ -1,9 +1,9 @@
1
1
  module ZmqJobs
2
2
  module Socket
3
- class Error < RuntimeError;end
3
+ Error = Class.new(StandardError)
4
4
  class Base
5
5
  attr_reader :socket, :options
6
-
6
+
7
7
  def initialize options={}
8
8
  iothreads = options['iothreads'] ? options['iothreads'].to_i : 1
9
9
  @context = options.delete('context') || ZMQ::Context.create(iothreads)
@@ -12,58 +12,68 @@ module ZmqJobs
12
12
  @socket = @context.socket(ZMQ::PUB)
13
13
  @running = false
14
14
  end
15
-
15
+
16
16
  def create_link url
17
17
  end
18
-
18
+
19
19
  def connect
20
20
  urls.each{|url|assert(create_link(url))}
21
21
  @connected = true
22
22
  self
23
23
  end
24
-
24
+
25
25
  def disconnect
26
26
  socket.close if connected?
27
27
  @connected = false
28
28
  self
29
29
  end
30
-
30
+
31
31
  def run
32
32
  @running = true
33
33
  connect
34
-
34
+
35
35
  loop do
36
- yield socket
36
+ yield self
37
37
  break unless running?
38
38
  end
39
39
 
40
40
  self
41
41
  end
42
-
42
+
43
43
  def stop
44
44
  @running = false
45
45
  disconnect
46
46
  end
47
-
47
+
48
48
  def terminate
49
49
  @context.terminate if @context && @context.respond_to?(:terminate)
50
50
  end
51
-
51
+
52
52
  def running?
53
53
  !!@running
54
54
  end
55
-
55
+
56
56
  def connected?
57
57
  !!@connected
58
58
  end
59
-
59
+
60
+ def send message
61
+ assert(socket.send_string(message))
62
+ end
63
+
64
+ def recv
65
+ message = ''
66
+ rc = socket.recv_string(message, ZMQ::NOBLOCK)
67
+ return message if ZMQ::Util.resultcode_ok?(rc)
68
+ end
69
+
60
70
  protected
61
71
  def assert(rc)
62
72
  raise Error.new(
63
73
  "Operation failed, errno [#{ZMQ::Util.errno}] description [#{ZMQ::Util.error_string}]"
64
74
  ) unless ZMQ::Util.resultcode_ok?(rc)
65
75
  end
66
-
76
+
67
77
  def urls
68
78
  u = []
69
79
  hosts.each do |host|
@@ -73,15 +83,15 @@ module ZmqJobs
73
83
  end
74
84
  u
75
85
  end
76
-
86
+
77
87
  def hosts
78
88
  options['hosts'].is_a?(Array) ? options['hosts'] : [options['hosts']]
79
89
  end
80
-
90
+
81
91
  def ports
82
92
  options['ports'].is_a?(Array) ? options['ports'] : [options['ports']]
83
93
  end
84
-
94
+
85
95
  def build_default_options
86
96
  {
87
97
  'linger' => 0,
@@ -1,3 +1,3 @@
1
1
  module ZmqJobs
2
- VERSION = "0.0.1.alpha2"
2
+ VERSION = "0.0.1.alpha3"
3
3
  end
@@ -1,6 +1,8 @@
1
1
  module ZmqJobs
2
2
  module Worker
3
3
  class Base
4
+ include ActiveSupport::Callbacks
5
+ define_callbacks :start, :stop, :execute
4
6
  attr_reader :options
5
7
  class << self
6
8
  def cmd args
@@ -18,32 +20,43 @@ module ZmqJobs
18
20
  end
19
21
 
20
22
  def initialize options={}
23
+ @debug = options.delete('debug') || false
21
24
  @options = options
22
25
  end
23
26
 
24
27
  def start
25
- trap('TERM'){stop}
26
- trap('INT'){stop}
28
+ trap('TERM'){stop;Process.exit}
29
+ trap('INT'){stop;Process.exit}
27
30
  logger.info "#{self.class} is starting ..."
31
+ logger.debug 'Debug mode' if debug?
28
32
 
29
- subscriber.run do |socket|
30
- message = ''
31
- rc = socket.recv_string(message, ZMQ::NOBLOCK)
32
- if ZMQ::Util.resultcode_ok?(rc)
33
- #message = BSON.deserialize(bson)
34
- execute_job message
33
+ run_callbacks :start do
34
+ subscriber.run do |socket|
35
+ message = socket.recv
36
+ execute_job message if message
37
+ sleep 1
35
38
  end
36
39
  end
37
40
  end
38
41
 
39
42
  def stop
40
- logger.info "Exiting..."
41
- subscriber.stop
42
- #subscriber.terminate
43
+ logger.info "Exiting..."
44
+ run_callbacks :stop do
45
+ subscriber.stop
46
+ #subscriber.terminate
47
+ end
43
48
  end
44
49
 
45
50
  def execute_job message
46
- execute(message)
51
+ run_callbacks :execute do
52
+ time = Time.now
53
+ logger.debug 'Start execute message...' if debug?
54
+ execute(message)
55
+ logger.debug "Stop execute message...Duration: #{Time.now - time}sec." if debug?
56
+ end
57
+ rescue => e
58
+ logger.warn format_exception_message(e)
59
+ #raise e
47
60
  end
48
61
 
49
62
  def subscriber
@@ -53,6 +66,17 @@ module ZmqJobs
53
66
  def logger
54
67
  ::ZmqJobs.logger
55
68
  end
69
+
70
+ def debug?
71
+ !!@debug
72
+ end
73
+
74
+ def format_exception_message exception
75
+ <<-MESSAGE
76
+ Execution error: #{exception} - #{exception.message}
77
+ #{exception.backtrace[0..10].join("\n ")}
78
+ MESSAGE
79
+ end
56
80
  end
57
81
  end
58
82
  end
@@ -1,7 +1,7 @@
1
1
  require 'spec_helper.rb'
2
2
 
3
3
  TestWorker = Class.new(ZmqJobs::Worker::Base)
4
-
4
+ ZmqJobs.logger = Logger.new('/dev/null')
5
5
  describe ZmqJobs::Worker::Base do
6
6
  describe '.cmd' do
7
7
  before do
@@ -11,7 +11,6 @@ describe ZmqJobs::Worker::Base do
11
11
 
12
12
  subject{TestWorker.cmd(options)}
13
13
 
14
-
15
14
  context 'without arguments' do
16
15
  let(:options){[]}
17
16
  let(:result_options){{}}
@@ -48,7 +47,17 @@ describe ZmqJobs::Worker::Base do
48
47
  end
49
48
  end
50
49
 
51
- describe '.new' do
50
+ describe '.start' do
51
+ before do
52
+ subscriber = mock(:subscriber)
53
+ subscriber.should_receive(:run).and_yield(worker).and_return(true)
54
+ worker.stub(:subscriber => subscriber)
55
+ worker.stub(:recv => 'Message')
56
+ worker.should_receive(:execute).with('Message').and_return(true)
57
+ end
58
+
59
+ let(:worker){TestWorker.new}
52
60
 
61
+ specify{worker.start.should be_true}
53
62
  end
54
63
  end
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: zmq_jobs
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease: 6
5
- version: 0.0.1.alpha2
5
+ version: 0.0.1.alpha3
6
6
  platform: ruby
7
7
  authors:
8
8
  - Undr
@@ -10,11 +10,12 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2011-11-11 00:00:00 +07:00
13
+ date: 2011-12-14 00:00:00 +07:00
14
14
  default_executable:
15
15
  dependencies:
16
16
  - !ruby/object:Gem::Dependency
17
17
  name: rspec
18
+ prerelease: false
18
19
  requirement: &id001 !ruby/object:Gem::Requirement
19
20
  none: false
20
21
  requirements:
@@ -22,10 +23,10 @@ dependencies:
22
23
  - !ruby/object:Gem::Version
23
24
  version: "2"
24
25
  type: :development
25
- prerelease: false
26
26
  version_requirements: *id001
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: yard
29
+ prerelease: false
29
30
  requirement: &id002 !ruby/object:Gem::Requirement
30
31
  none: false
31
32
  requirements:
@@ -33,10 +34,10 @@ dependencies:
33
34
  - !ruby/object:Gem::Version
34
35
  version: 0.6.0
35
36
  type: :development
36
- prerelease: false
37
37
  version_requirements: *id002
38
38
  - !ruby/object:Gem::Dependency
39
39
  name: ruby-debug19
40
+ prerelease: false
40
41
  requirement: &id003 !ruby/object:Gem::Requirement
41
42
  none: false
42
43
  requirements:
@@ -44,10 +45,10 @@ dependencies:
44
45
  - !ruby/object:Gem::Version
45
46
  version: "0"
46
47
  type: :development
47
- prerelease: false
48
48
  version_requirements: *id003
49
49
  - !ruby/object:Gem::Dependency
50
50
  name: bundler
51
+ prerelease: false
51
52
  requirement: &id004 !ruby/object:Gem::Requirement
52
53
  none: false
53
54
  requirements:
@@ -55,10 +56,10 @@ dependencies:
55
56
  - !ruby/object:Gem::Version
56
57
  version: 1.0.0
57
58
  type: :runtime
58
- prerelease: false
59
59
  version_requirements: *id004
60
60
  - !ruby/object:Gem::Dependency
61
61
  name: rake
62
+ prerelease: false
62
63
  requirement: &id005 !ruby/object:Gem::Requirement
63
64
  none: false
64
65
  requirements:
@@ -66,10 +67,10 @@ dependencies:
66
67
  - !ruby/object:Gem::Version
67
68
  version: "0"
68
69
  type: :runtime
69
- prerelease: false
70
70
  version_requirements: *id005
71
71
  - !ruby/object:Gem::Dependency
72
72
  name: ffi
73
+ prerelease: false
73
74
  requirement: &id006 !ruby/object:Gem::Requirement
74
75
  none: false
75
76
  requirements:
@@ -77,10 +78,10 @@ dependencies:
77
78
  - !ruby/object:Gem::Version
78
79
  version: "0"
79
80
  type: :runtime
80
- prerelease: false
81
81
  version_requirements: *id006
82
82
  - !ruby/object:Gem::Dependency
83
83
  name: ffi-rzmq
84
+ prerelease: false
84
85
  requirement: &id007 !ruby/object:Gem::Requirement
85
86
  none: false
86
87
  requirements:
@@ -88,10 +89,10 @@ dependencies:
88
89
  - !ruby/object:Gem::Version
89
90
  version: "0"
90
91
  type: :runtime
91
- prerelease: false
92
92
  version_requirements: *id007
93
93
  - !ruby/object:Gem::Dependency
94
94
  name: activesupport
95
+ prerelease: false
95
96
  requirement: &id008 !ruby/object:Gem::Requirement
96
97
  none: false
97
98
  requirements:
@@ -99,10 +100,10 @@ dependencies:
99
100
  - !ruby/object:Gem::Version
100
101
  version: "0"
101
102
  type: :runtime
102
- prerelease: false
103
103
  version_requirements: *id008
104
104
  - !ruby/object:Gem::Dependency
105
105
  name: daemons
106
+ prerelease: false
106
107
  requirement: &id009 !ruby/object:Gem::Requirement
107
108
  none: false
108
109
  requirements:
@@ -110,7 +111,6 @@ dependencies:
110
111
  - !ruby/object:Gem::Version
111
112
  version: "0"
112
113
  type: :runtime
113
- prerelease: false
114
114
  version_requirements: *id009
115
115
  description: ZeroMQ-based queue system for background job
116
116
  email:
@@ -136,6 +136,7 @@ files:
136
136
  - examples/workers/default.rb
137
137
  - examples/workers/test_worker.rb
138
138
  - lib/command.rb
139
+ - lib/rails/railties.rb
139
140
  - lib/zmq_jobs.rb
140
141
  - lib/zmq_jobs/devices/balancer.rb
141
142
  - lib/zmq_jobs/devices/broker.rb
@@ -163,9 +164,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
163
164
  requirements:
164
165
  - - ">="
165
166
  - !ruby/object:Gem::Version
166
- hash: 2610017728678870612
167
- segments:
168
- - 0
169
167
  version: "0"
170
168
  required_rubygems_version: !ruby/object:Gem::Requirement
171
169
  none: false