sqskiq 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/sqskiq/aws.rb ADDED
@@ -0,0 +1,20 @@
1
+ require 'aws-sdk'
2
+
3
+ module Sqskiq
4
+ module AWS
5
+
6
+ def init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
7
+ sqs = ::AWS::SQS.new(:access_key_id => aws_access_key_id, :secret_access_key => aws_secret_access_key)
8
+ @queue = sqs.queues.named(queue_name.to_s)
9
+ end
10
+
11
+ def fetch_sqs_messages
12
+ @queue.receive_message(:limit => 10, :attributes => [:receive_count])
13
+ end
14
+
15
+ def delete_sqs_messages(messages)
16
+ @queue.batch_delete(messages)
17
+ end
18
+
19
+ end
20
+ end
@@ -0,0 +1,46 @@
1
+ require 'celluloid'
2
+ require 'celluloid/autostart'
3
+
4
+ module Sqskiq
5
+ class BatchProcessor
6
+ include Celluloid
7
+ include Celluloid::Notifications
8
+
9
+ def initialize
10
+ @manager = Celluloid::Actor[:manager]
11
+ @processor = Celluloid::Actor[:processor]
12
+
13
+ subscribe_interrupt
14
+ end
15
+
16
+ def batch_process(messages)
17
+ p "processing #{messages.size} messages"
18
+
19
+ process_result = []
20
+ messages.each do |message|
21
+ process_result << @processor.future.process(message)
22
+ end
23
+
24
+ success_messages = []
25
+ process_result.each do |result|
26
+ value = result.value
27
+ if value[:success]
28
+ success_messages << value[:message]
29
+ end
30
+ end
31
+
32
+ @manager.async.batch_process_done(success_messages)
33
+ end
34
+
35
+ def subscribe_interrupt
36
+ subscribe('SIGINT', :interrupt)
37
+ subscribe('TERM', :interrupt)
38
+ subscribe('SIGTERM', :interrupt)
39
+ end
40
+
41
+ def interrupt(signal)
42
+ self.terminate
43
+ end
44
+
45
+ end
46
+ end
@@ -0,0 +1,19 @@
1
+ require 'celluloid'
2
+ require 'celluloid/autostart'
3
+ require 'sqskiq/aws'
4
+
5
+ module Sqskiq
6
+ class Deleter
7
+ include Celluloid
8
+ include Celluloid::Notifications
9
+ include Sqskiq::AWS
10
+
11
+ def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
12
+ init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
13
+ end
14
+
15
+ def delete(messages)
16
+ delete_sqs_messages(messages) if not messages.empty?
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,34 @@
1
+ require 'celluloid'
2
+ require 'celluloid/autostart'
3
+ require 'sqskiq/aws'
4
+
5
+ module Sqskiq
6
+ class Fetcher
7
+ include Celluloid
8
+ include Celluloid::Notifications
9
+ include Sqskiq::AWS
10
+
11
+ def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
12
+ init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
13
+ subscribe_interrupt
14
+
15
+ @manager = Celluloid::Actor[:manager]
16
+ end
17
+
18
+ def fetch
19
+ messages = fetch_sqs_messages
20
+ @manager.async.fetch_done(messages)
21
+ end
22
+
23
+ def subscribe_interrupt
24
+ subscribe('SIGINT', :interrupt)
25
+ subscribe('SIGTERM', :interrupt)
26
+ subscribe('TERM', :interrupt)
27
+ end
28
+
29
+ def interrupt(signal)
30
+ self.terminate
31
+ end
32
+
33
+ end
34
+ end
@@ -0,0 +1,50 @@
1
+ require 'celluloid'
2
+ require 'celluloid/autostart'
3
+
4
+
5
+ module Sqskiq
6
+ class Manager
7
+ include Celluloid
8
+ include Celluloid::Notifications
9
+
10
+ def initialize
11
+ @shutting_down = false
12
+ subscribe_shutting_down
13
+ end
14
+
15
+ def bootstrap
16
+ @fetcher = Celluloid::Actor[:fetcher]
17
+ @batch_processor = Celluloid::Actor[:batch_processor]
18
+ @deleter = Celluloid::Actor[:deleter]
19
+ #TODO default value. Should be configurable
20
+ new_fetch(2)
21
+ end
22
+
23
+ def fetch_done(messages)
24
+ @batch_processor.async.batch_process(messages) unless @shutting_down
25
+ end
26
+
27
+ def batch_process_done(messages)
28
+ @deleter.async.delete(messages) if @deleter.alive?
29
+ new_fetch(1)
30
+ end
31
+
32
+ def new_fetch(num)
33
+ num.times { @fetcher.async.fetch unless @shutting_down }
34
+ end
35
+
36
+ def shutting_down(signal)
37
+ @shutting_down = true
38
+ end
39
+
40
+ def running?
41
+ not (@deleter.busy_size == 0 and @shutting_down)
42
+ end
43
+
44
+ def subscribe_shutting_down
45
+ subscribe('SIGINT', :shutting_down)
46
+ subscribe('TERM', :shutting_down)
47
+ subscribe('SIGTERM', :shutting_down)
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,36 @@
1
+ require 'celluloid'
2
+ require 'celluloid/autostart'
3
+
4
+ module Sqskiq
5
+ class Processor
6
+ include Celluloid
7
+ include Celluloid::Notifications
8
+
9
+ def initialize(worker_class)
10
+ @worker_instance = worker_class.new
11
+
12
+ subscribe_interrupt
13
+ end
14
+
15
+ def process(message)
16
+ result = true
17
+ begin
18
+ @worker_instance.perform(message)
19
+ rescue Exception => e
20
+ result = false
21
+ end
22
+ { :success => result, :message => message }
23
+ end
24
+
25
+ def subscribe_interrupt
26
+ subscribe('SIGINT', :interrupt)
27
+ subscribe('TERM', :interrupt)
28
+ subscribe('SIGTERM', :interrupt)
29
+ end
30
+
31
+ def interrupt(signal)
32
+ self.terminate
33
+ end
34
+
35
+ end
36
+ end
@@ -0,0 +1,21 @@
1
+ module Sqskiq
2
+ module Worker
3
+
4
+ module ClassMethods
5
+ def run
6
+ Sqskiq.bootstrap(sqskiq_options_hash, self)
7
+ end
8
+
9
+ def sqskiq_options(options)
10
+ self.sqskiq_options_hash = options
11
+ end
12
+ end
13
+
14
+ def self.included(base)
15
+ base.send :extend, ClassMethods
16
+ base.class_attribute :sqskiq_options_hash
17
+ end
18
+
19
+ end
20
+
21
+ end
data/lib/sqskiq.rb ADDED
@@ -0,0 +1,59 @@
1
+ require "sqskiq/manager"
2
+ require 'sqskiq/fetch'
3
+ require 'sqskiq/process'
4
+ require 'sqskiq/delete'
5
+ require 'sqskiq/worker'
6
+ require 'sqskiq/batch_process'
7
+
8
+ module Sqskiq
9
+
10
+ def self.bootstrap(options, worker_class)
11
+
12
+ params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
13
+
14
+ Celluloid::Actor[:manager] = @manager = Manager.new
15
+ Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => 2, :args => params)
16
+ Celluloid::Actor[:processor] = @processor = Processor.pool(:size => 20, :args => worker_class)
17
+ Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => 2)
18
+ Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => 2, :args => params)
19
+
20
+ p "pid = #{Process.pid}"
21
+
22
+ trap('SIGTERM') do
23
+ @manager.publish('SIGTERM')
24
+ end
25
+
26
+ trap('TERM') do
27
+ @manager.publish('TERM')
28
+ end
29
+
30
+ trap('SIGINT') do
31
+ @manager.publish('SIGINT')
32
+ end
33
+
34
+ @manager.bootstrap
35
+ while @manager.running? do
36
+ sleep 2
37
+ end
38
+
39
+ @fetcher.__shutdown__
40
+ @batch_processor.__shutdown__
41
+ @processor.__shutdown__
42
+ @deleter.__shutdown__
43
+
44
+ @manager.terminate
45
+ end
46
+
47
+ def self.configure
48
+ yield self
49
+ end
50
+
51
+ def self.aws_access_key_id=(value)
52
+ @aws_access_key_id = value
53
+ end
54
+
55
+ def self.aws_secret_access_key=(value)
56
+ @aws_secret_access_key = value
57
+ end
58
+
59
+ end
metadata ADDED
@@ -0,0 +1,84 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sqskiq
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Ricardo Lazaro de Oliveira
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2013-06-01 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: celluloid
16
+ requirement: !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ~>
20
+ - !ruby/object:Gem::Version
21
+ version: 0.14.1
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ~>
28
+ - !ruby/object:Gem::Version
29
+ version: 0.14.1
30
+ - !ruby/object:Gem::Dependency
31
+ name: aws-sdk
32
+ requirement: !ruby/object:Gem::Requirement
33
+ none: false
34
+ requirements:
35
+ - - ~>
36
+ - !ruby/object:Gem::Version
37
+ version: 1.9.1
38
+ type: :runtime
39
+ prerelease: false
40
+ version_requirements: !ruby/object:Gem::Requirement
41
+ none: false
42
+ requirements:
43
+ - - ~>
44
+ - !ruby/object:Gem::Version
45
+ version: 1.9.1
46
+ description: sqskiq
47
+ email: ri.vanlazar@gmail.com
48
+ executables: []
49
+ extensions: []
50
+ extra_rdoc_files: []
51
+ files:
52
+ - lib/sqskiq.rb
53
+ - lib/sqskiq/manager.rb
54
+ - lib/sqskiq/fetch.rb
55
+ - lib/sqskiq/process.rb
56
+ - lib/sqskiq/delete.rb
57
+ - lib/sqskiq/worker.rb
58
+ - lib/sqskiq/batch_process.rb
59
+ - lib/sqskiq/aws.rb
60
+ homepage: http://rubygems.org/gems/sqskiq
61
+ licenses: []
62
+ post_install_message:
63
+ rdoc_options: []
64
+ require_paths:
65
+ - lib
66
+ required_ruby_version: !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - ! '>='
70
+ - !ruby/object:Gem::Version
71
+ version: '0'
72
+ required_rubygems_version: !ruby/object:Gem::Requirement
73
+ none: false
74
+ requirements:
75
+ - - ! '>='
76
+ - !ruby/object:Gem::Version
77
+ version: '0'
78
+ requirements: []
79
+ rubyforge_project:
80
+ rubygems_version: 1.8.25
81
+ signing_key:
82
+ specification_version: 3
83
+ summary: sqskiq
84
+ test_files: []