boxxspring-workers 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/boxxspring/abstract.rb +37 -0
- data/lib/boxxspring/journal.rb +60 -0
- data/lib/boxxspring/synchronization/configuration.rb +32 -0
- data/lib/boxxspring/synchronization/mutex.rb +24 -0
- data/lib/boxxspring/synchronization/operations.rb +81 -0
- data/lib/boxxspring/synchronization/orchestrator.rb +74 -0
- data/lib/boxxspring/synchronization/variable.rb +26 -0
- data/lib/boxxspring/synchronization.rb +17 -0
- data/lib/boxxspring/worker/base.rb +185 -0
- data/lib/boxxspring/worker/configuration.rb +41 -0
- data/lib/boxxspring/worker/task_base.rb +151 -0
- data/lib/boxxspring-worker-version.rb +5 -0
- data/lib/boxxspring-worker.rb +15 -0
- data/lib/tasks/workers.rake +38 -0
- metadata +142 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5a8ab0b25057afe43c90af53902ec1410d5230c2
|
4
|
+
data.tar.gz: f57b1b142e88bac02e52d893f1824fd63e19cacf
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 141d7d76f6209efe1b7beb56673094e1ed549dbf572dfef923524b4ab295ea35741b6961e4b655a612b91b0f52dabd8f81c47674fc74ca565b3f1c884e7d0e0a
|
7
|
+
data.tar.gz: 3f135b6596edf3a741e9df12d2870a9eafa8a3aded85205c198e17e94a2f651056452de89213be800bf81a54ed5bcfbb0ab38df2cb665786d91cb847b3a43abc
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
class Abstract
|
4
|
+
|
5
|
+
def initialize( attributes = {} )
|
6
|
+
@attributes = {}
|
7
|
+
attributes.each_pair do | key, value |
|
8
|
+
self.send( key, value )
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def to_hash
|
13
|
+
return @attributes
|
14
|
+
end
|
15
|
+
|
16
|
+
def method_missing( method, *arguments, &block )
|
17
|
+
result = nil
|
18
|
+
if arguments.length == 0
|
19
|
+
result = @attributes[ method.to_sym ]
|
20
|
+
if result.nil?
|
21
|
+
result = Abstract.new
|
22
|
+
result.instance_eval( &block ) unless block.nil?
|
23
|
+
@attributes[ method.to_sym ] = result
|
24
|
+
end
|
25
|
+
elsif arguments.length == 1
|
26
|
+
method = method.to_s.gsub( /=$/, '' )
|
27
|
+
result = arguments[ 0 ]
|
28
|
+
@attributes[ method.to_sym ] = result
|
29
|
+
else
|
30
|
+
result = super
|
31
|
+
end
|
32
|
+
result
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
class Journal
|
4
|
+
|
5
|
+
def initialize( name )
|
6
|
+
@db_name = name
|
7
|
+
end
|
8
|
+
|
9
|
+
def write( id, attributes )
|
10
|
+
db_attributes = []
|
11
|
+
attributes.each_pair do | key, value |
|
12
|
+
db_attributes.push( {
|
13
|
+
name: key.to_s, value: value.to_s, replace: true
|
14
|
+
} )
|
15
|
+
end
|
16
|
+
self.db.put_attributes( {
|
17
|
+
domain_name: @db_name,
|
18
|
+
item_name: id.to_s,
|
19
|
+
attributes: db_attributes
|
20
|
+
} )
|
21
|
+
end
|
22
|
+
|
23
|
+
def read( id )
|
24
|
+
result = nil
|
25
|
+
db_record = self.db.get_attributes(
|
26
|
+
domain_name: @db_name,
|
27
|
+
item_name: id.to_s,
|
28
|
+
consistent_read: true
|
29
|
+
)
|
30
|
+
if db_record.present? &&
|
31
|
+
db_record.attributes.present? &&
|
32
|
+
db_record.attributes.length > 0
|
33
|
+
result = {}
|
34
|
+
db_record.attributes.each do | attribute |
|
35
|
+
result[ attribute.name.to_sym ] = attribute.value
|
36
|
+
end
|
37
|
+
end
|
38
|
+
result
|
39
|
+
end
|
40
|
+
|
41
|
+
def delete( id )
|
42
|
+
db_record = self.db.delete_attributes(
|
43
|
+
domain_name: @db_name,
|
44
|
+
item_name: id.to_s
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
protected; def db
|
49
|
+
@db ||= begin
|
50
|
+
db = Aws::SimpleDB::Client.new
|
51
|
+
db.create_domain(
|
52
|
+
domain_name: @db_name
|
53
|
+
)
|
54
|
+
db
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Synchronization
|
4
|
+
|
5
|
+
class Configuration
|
6
|
+
|
7
|
+
include Singleton
|
8
|
+
|
9
|
+
def self.field( field_name, options = {} )
|
10
|
+
|
11
|
+
class_eval(
|
12
|
+
"def #{field_name}( *arguments ); " +
|
13
|
+
"@#{field_name} = arguments.first unless arguments.empty?; " +
|
14
|
+
"@#{field_name} || " +
|
15
|
+
( options[ :default ].blank? ?
|
16
|
+
"nil" :
|
17
|
+
( options[ :default ].is_a?( String ) ?
|
18
|
+
"'#{options[ :default ]}'" :
|
19
|
+
"#{options[ :default ]}" ) ) + ";" +
|
20
|
+
"end",
|
21
|
+
__FILE__,
|
22
|
+
__LINE__
|
23
|
+
)
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
field :url, default: "redis://127.0.0.1:6379/0"
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
class Mutex
|
5
|
+
|
6
|
+
def initialize( name, signature = nil )
|
7
|
+
@orchestrator = Synchronization::Orchestrator.instance
|
8
|
+
@name = name
|
9
|
+
@signature = signature || SecureRandom.hex
|
10
|
+
end
|
11
|
+
|
12
|
+
def lock( options = {} )
|
13
|
+
@orchestrator.lock( @name, @signature, options )
|
14
|
+
end
|
15
|
+
|
16
|
+
def unlock
|
17
|
+
@orchestrator.unlock( @name, @signature )
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
self.const_set 'Operations', {
|
5
|
+
|
6
|
+
lock:
|
7
|
+
"local result = false; "\
|
8
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
9
|
+
"if ( not value ) then "\
|
10
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
11
|
+
"result = true; "\
|
12
|
+
"elseif value == ARGV[ 1 ] then "\
|
13
|
+
"result = true; "\
|
14
|
+
"end; "\
|
15
|
+
"if result == true and not ( ARGV[ 2 ] == '' ) then "\
|
16
|
+
"redis.call( 'pexpire', KEYS[ 1 ], ARGV[ 2 ] ); "\
|
17
|
+
"end; "\
|
18
|
+
"return result",
|
19
|
+
|
20
|
+
unlock:
|
21
|
+
"local result = false; "\
|
22
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
23
|
+
"if ( not value ) then "\
|
24
|
+
"result = true; "\
|
25
|
+
"elseif value == ARGV[ 1 ] then "\
|
26
|
+
"redis.call( 'del', KEYS[ 1 ] ); "\
|
27
|
+
"result = true; "\
|
28
|
+
"else "\
|
29
|
+
"result = false; "\
|
30
|
+
"end; " \
|
31
|
+
"return result",
|
32
|
+
|
33
|
+
write_if_greater_than:
|
34
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
35
|
+
"if ( not value ) or ( ARGV[ 1 ] > value ) then "\
|
36
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
37
|
+
"return true; "\
|
38
|
+
"else "\
|
39
|
+
"return false;"\
|
40
|
+
"end",
|
41
|
+
|
42
|
+
write_if_greater_than_or_equal_to:
|
43
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
44
|
+
"if ( not value ) or ( ARGV[ 1 ] >= value ) then "\
|
45
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
46
|
+
"return true; "\
|
47
|
+
"else "\
|
48
|
+
"return false;"\
|
49
|
+
"end",
|
50
|
+
|
51
|
+
write_if_equal_to:
|
52
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
53
|
+
"if ( not value ) or ( ARGV[ 1 ] == value ) then "\
|
54
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
55
|
+
"return true; "\
|
56
|
+
"else "\
|
57
|
+
"return false;"\
|
58
|
+
"end",
|
59
|
+
|
60
|
+
write_if_less_than_or_equal_to:
|
61
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
62
|
+
"if ( not value ) or ( ARGV[ 1 ] <= value ) then "\
|
63
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
64
|
+
"return true; "\
|
65
|
+
"else "\
|
66
|
+
"return false;"\
|
67
|
+
"end",
|
68
|
+
|
69
|
+
write_if_less_than:
|
70
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
71
|
+
"if ( not value ) or ( ARGV[ 1 ] < value ) then "\
|
72
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
73
|
+
"return true; "\
|
74
|
+
"else "\
|
75
|
+
"return false;"\
|
76
|
+
"end"
|
77
|
+
|
78
|
+
}
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
module Boxxspring
|
4
|
+
module Synchronization
|
5
|
+
|
6
|
+
class Orchestrator
|
7
|
+
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@provider = ::Redis.new(
|
12
|
+
url: Synchronization.configuration.url,
|
13
|
+
timeout: 10.0
|
14
|
+
)
|
15
|
+
@operations = {}
|
16
|
+
end
|
17
|
+
|
18
|
+
def lock( key, signature, options = {} )
|
19
|
+
ttl = options[ :ttl ];
|
20
|
+
ttl = ttl * 1000 if ttl.is_a? ActiveSupport::Duration
|
21
|
+
self.execute_operation( :lock, [ key ], [ signature, ttl ] ) ?
|
22
|
+
true : false
|
23
|
+
end
|
24
|
+
|
25
|
+
def unlock( key, signature )
|
26
|
+
self.execute_operation( :unlock, [ key ], [ signature ] ) ?
|
27
|
+
true : false
|
28
|
+
end
|
29
|
+
|
30
|
+
def read( key )
|
31
|
+
@provider.get( key )
|
32
|
+
end
|
33
|
+
|
34
|
+
def write( key, value, options = {} )
|
35
|
+
ttl = options[ :ttl ]
|
36
|
+
if ttl
|
37
|
+
ttl = ttl.to_i if ttl.is_a? ActiveSupport::Duration
|
38
|
+
( @provider.set( key, value ) == "OK" ) &&
|
39
|
+
( @provider.expire( key, ttl ) ) ? true : false
|
40
|
+
else
|
41
|
+
( @provider.set( key, value ) == "OK" ) ? true : false
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def write_if_condition( key, value, condition )
|
46
|
+
operation = "write_if_#{condition}"
|
47
|
+
operation_sha = @operations[ operation.to_sym ]
|
48
|
+
raise 'Synchronization: An unknown condition was requested.' \
|
49
|
+
if operation_sha.nil?
|
50
|
+
@provider.evalsha( operation_sha, [ key ], [ value ] ) ? true : false
|
51
|
+
end
|
52
|
+
|
53
|
+
protected; def execute_operation( operation, keys, arguments )
|
54
|
+
try ||= 1
|
55
|
+
sha = @operations[ operation ] || install_operation( operation )
|
56
|
+
@provider.evalsha( sha, keys, arguments ) ? true : false
|
57
|
+
rescue Redis::CommandError => error
|
58
|
+
raise error unless error.message.match( /\ANOSCRIPT/ )
|
59
|
+
self.install_operation( operation )
|
60
|
+
retry if ( try -= 1 ) >= 0
|
61
|
+
end
|
62
|
+
|
63
|
+
protected; def install_operation( operation )
|
64
|
+
@operations[ operation ] = @provider.script(
|
65
|
+
:load,
|
66
|
+
Operations[ operation ]
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
class Variable
|
5
|
+
|
6
|
+
def initialize( name )
|
7
|
+
@orchestrator = Synchronization::Orchestrator.instance
|
8
|
+
@name = name
|
9
|
+
end
|
10
|
+
|
11
|
+
def read
|
12
|
+
@orchestrator.read( @name )
|
13
|
+
end
|
14
|
+
|
15
|
+
def write( value, options={} )
|
16
|
+
@orchestrator.write( @name, value, options )
|
17
|
+
end
|
18
|
+
|
19
|
+
def write_if_condition( value, condition )
|
20
|
+
@orchestrator.write_if_condition( @name, value, condition )
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#
|
2
|
+
# module. Synchronization
|
3
|
+
#
|
4
|
+
# The synchornization module implements lock/unlock and key/value read/write
|
5
|
+
# operations accross multiple instances of the application.
|
6
|
+
|
7
|
+
module Boxxspring
|
8
|
+
module Synchronization
|
9
|
+
|
10
|
+
def self.configuration( &block )
|
11
|
+
configuration = Synchronization::Configuration.instance
|
12
|
+
configuration.instance_eval( &block ) unless block.nil?
|
13
|
+
configuration
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,185 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Worker
|
4
|
+
|
5
|
+
QUEUE_MESSAGE_REQUEST_COUNT = 10
|
6
|
+
QUEUE_MESSAGE_WAIT_IN_SECONDS = 4
|
7
|
+
|
8
|
+
class Base
|
9
|
+
|
10
|
+
#------------------------------------------------------------------------
|
11
|
+
# class attributes
|
12
|
+
|
13
|
+
class_attribute :queue_name
|
14
|
+
class_attribute :processor
|
15
|
+
|
16
|
+
#------------------------------------------------------------------------
|
17
|
+
# class methods
|
18
|
+
|
19
|
+
class << self
|
20
|
+
|
21
|
+
public; def process( &block )
|
22
|
+
self.processor = block
|
23
|
+
end
|
24
|
+
|
25
|
+
def queue_interface
|
26
|
+
@queue_interface ||= Aws::SQS::Client.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def queue_url
|
30
|
+
unless @queue_url.present?
|
31
|
+
response = self.queue_interface.create_queue(
|
32
|
+
queue_name: self.full_queue_name
|
33
|
+
)
|
34
|
+
@queue_url = response[ :queue_url ]
|
35
|
+
end
|
36
|
+
@queue_url
|
37
|
+
end
|
38
|
+
|
39
|
+
protected; def full_queue_name
|
40
|
+
name = ( Worker.env == 'development' ) ?
|
41
|
+
( ENV[ 'USER' ] || 'development' ) :
|
42
|
+
Worker.env
|
43
|
+
name += '-' +
|
44
|
+
( self.queue_name ||
|
45
|
+
self.name.
|
46
|
+
underscore.
|
47
|
+
gsub( /[\/]/, '-' ).
|
48
|
+
gsub( /_worker\Z/, '' ) )
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
#------------------------------------------------------------------------
|
54
|
+
# operations
|
55
|
+
|
56
|
+
def process
|
57
|
+
messages = self.receive_messages() || []
|
58
|
+
messages.each do | message |
|
59
|
+
if message.present?
|
60
|
+
payload = self.payload_from_message( message )
|
61
|
+
if payload.present?
|
62
|
+
begin
|
63
|
+
result = self.process_payload( payload )
|
64
|
+
# note: if an exception is raised the message will be deleted
|
65
|
+
self.delete_message( message ) unless result == false
|
66
|
+
rescue StandardError => error
|
67
|
+
self.logger.error(
|
68
|
+
"The #{ self.human_name } worker failed to process the " +
|
69
|
+
"payload. "
|
70
|
+
)
|
71
|
+
self.logger.error( error.message )
|
72
|
+
self.logger.error( error.backtrace.join( "\n" ) )
|
73
|
+
end
|
74
|
+
else
|
75
|
+
# note: messages with invalid payloads are deleted
|
76
|
+
self.delete_message( message )
|
77
|
+
self.logger.error(
|
78
|
+
"The #{ self.human_name } worker received an invalid payload."
|
79
|
+
)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
protected; def logger
|
86
|
+
@logger ||= Boxxspring::Worker.configuration.logger
|
87
|
+
end
|
88
|
+
|
89
|
+
#------------------------------------------------------------------------
|
90
|
+
# implementation
|
91
|
+
|
92
|
+
protected; def receive_messages
|
93
|
+
messages = nil
|
94
|
+
begin
|
95
|
+
response = self.class.queue_interface.receive_message(
|
96
|
+
queue_url: self.class.queue_url,
|
97
|
+
max_number_of_messages: QUEUE_MESSAGE_REQUEST_COUNT,
|
98
|
+
wait_time_seconds: QUEUE_MESSAGE_WAIT_IN_SECONDS
|
99
|
+
)
|
100
|
+
messages = response[ :messages ]
|
101
|
+
rescue StandardError => error
|
102
|
+
raise RuntimeError.new(
|
103
|
+
"The #{ self.human_name } worker is unable to receive a message " +
|
104
|
+
"from the queue. #{error.message}."
|
105
|
+
)
|
106
|
+
end
|
107
|
+
messages
|
108
|
+
end
|
109
|
+
|
110
|
+
protected; def delete_message( message )
|
111
|
+
begin
|
112
|
+
self.class.queue_interface.delete_message(
|
113
|
+
queue_url: self.class.queue_url,
|
114
|
+
receipt_handle: message[ :receipt_handle ]
|
115
|
+
)
|
116
|
+
rescue StandardError => error
|
117
|
+
raise RuntimeError.new(
|
118
|
+
"The #{ self.human_name } worker is unable to delete the " +
|
119
|
+
"message from the queue. #{error.message}."
|
120
|
+
)
|
121
|
+
end
|
122
|
+
message
|
123
|
+
end
|
124
|
+
|
125
|
+
protected; def payload_from_message( message )
|
126
|
+
payload = message.body
|
127
|
+
if ( payload.present? )
|
128
|
+
payload = JSON.parse( payload ) rescue payload
|
129
|
+
if ( payload.is_a?( Hash ) &&
|
130
|
+
payload.include?( 'Type' ) &&
|
131
|
+
payload[ 'Type' ] == 'Notification' )
|
132
|
+
payload = payload[ 'Message' ]
|
133
|
+
payload = payload.present? ?
|
134
|
+
( JSON.parse( payload ) rescue payload ) :
|
135
|
+
payload
|
136
|
+
end
|
137
|
+
end
|
138
|
+
payload
|
139
|
+
end
|
140
|
+
|
141
|
+
protected; def process_payload( payload )
|
142
|
+
if self.class.processor.present?
|
143
|
+
self.class.processor.call( payload )
|
144
|
+
else
|
145
|
+
raise RuntimeError.new(
|
146
|
+
"The worker lacks a processor"
|
147
|
+
)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
protected; def delegate_payload( queue_name, payload )
|
152
|
+
queue_name_prefix = ( Worker.env == 'development' ) ?
|
153
|
+
( ENV[ 'USER' ] || 'development' ) : Worker.env
|
154
|
+
queue_name = queue_name_prefix + '-' + queue_name
|
155
|
+
begin
|
156
|
+
response = self.class.queue_interface.create_queue(
|
157
|
+
queue_name: queue_name
|
158
|
+
)
|
159
|
+
queue_url = response[ :queue_url ]
|
160
|
+
if queue_url.present?
|
161
|
+
self.class.queue_interface.send_message(
|
162
|
+
queue_url: queue_url,
|
163
|
+
message_body: payload.to_json
|
164
|
+
)
|
165
|
+
end
|
166
|
+
rescue StandardError => error
|
167
|
+
raise RuntimeError.new(
|
168
|
+
"The #{ self.human_name } worker was unable to delegate the " +
|
169
|
+
"payload to the queue name '#{ queue_name }'. #{error.message}."
|
170
|
+
)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
protected; def human_name
|
175
|
+
self.class.name.
|
176
|
+
underscore.
|
177
|
+
gsub( /[\/]/, ' ' ).
|
178
|
+
gsub( /_worker\Z/, '' )
|
179
|
+
end
|
180
|
+
|
181
|
+
end
|
182
|
+
|
183
|
+
end
|
184
|
+
|
185
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Boxxspring
|
4
|
+
|
5
|
+
module Worker
|
6
|
+
|
7
|
+
def self.configuration( &block )
|
8
|
+
Configuration.instance().instance_eval( &block ) unless block.nil?
|
9
|
+
Configuration.instance()
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.env
|
13
|
+
self.configuration.env
|
14
|
+
end
|
15
|
+
|
16
|
+
class Configuration < Abstract
|
17
|
+
|
18
|
+
include Singleton
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
super( {
|
22
|
+
env: ENV[ 'WORKERS_ENV' ] || 'development',
|
23
|
+
logger: Logger.new( STDOUT )
|
24
|
+
} )
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.reloadable?
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
def from_hash( configuration )
|
32
|
+
configuration.each_pair do | name, value |
|
33
|
+
self.send( "@#{name}", value )
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Worker
|
4
|
+
|
5
|
+
class TaskBase < Base
|
6
|
+
|
7
|
+
#------------------------------------------------------------------------
|
8
|
+
# class attributes
|
9
|
+
|
10
|
+
class_attribute :task_type_name
|
11
|
+
class_attribute :task_state
|
12
|
+
|
13
|
+
#------------------------------------------------------------------------
|
14
|
+
# operations
|
15
|
+
|
16
|
+
protected; def process_task( task )
|
17
|
+
if self.class.processor.present?
|
18
|
+
self.class.processor.call( task )
|
19
|
+
else
|
20
|
+
raise RuntimeError.new(
|
21
|
+
"The #{self.human_name} worker lacks a task processor"
|
22
|
+
)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
#------------------------------------------------------------------------
|
27
|
+
# implementation
|
28
|
+
|
29
|
+
protected; def process_payload( payload )
|
30
|
+
|
31
|
+
result = true
|
32
|
+
type_names = self.task_type_name.blank? ?
|
33
|
+
nil :
|
34
|
+
[ self.task_type_name ].flatten
|
35
|
+
states = self.task_state.blank? ?
|
36
|
+
nil :
|
37
|
+
[ self.task_state ].flatten
|
38
|
+
|
39
|
+
tasks = payload[ 'tasks' ]
|
40
|
+
if ( tasks.present? && tasks.respond_to?( :each ) )
|
41
|
+
tasks.each do | task |
|
42
|
+
task_id = task[ 'id' ]
|
43
|
+
if ( type_names.blank? || type_names.include?( task[ 'type_name' ] ) )
|
44
|
+
task = task_read( task[ 'property_id' ], task_id )
|
45
|
+
if task.is_a?( Boxxspring::Task )
|
46
|
+
if ( states.blank? || states.include?( task.state ) )
|
47
|
+
self.logger.info(
|
48
|
+
"The task (id: #{task.id}) processing has started."
|
49
|
+
)
|
50
|
+
begin
|
51
|
+
result = self.process_task( task )
|
52
|
+
message = "The task (id: #{task.id}) processing has ended"
|
53
|
+
message += " and the message has been retained." if result == false
|
54
|
+
self.logger.info(
|
55
|
+
message
|
56
|
+
)
|
57
|
+
rescue SignalException, StandardError => error
|
58
|
+
if error.is_a?( SignalException )
|
59
|
+
task_state = 'idle'
|
60
|
+
task_message = "The task (id: #{task.id}) has restarted."
|
61
|
+
else
|
62
|
+
task_state = 'failed'
|
63
|
+
task_message = "The task (id: #{task.id}) processing has failed."
|
64
|
+
end
|
65
|
+
task = task_write_state(task,
|
66
|
+
task_state,
|
67
|
+
task_message)
|
68
|
+
self.logger.error( error.message )
|
69
|
+
self.logger.error( error.backtrace.join( "\n" ) )
|
70
|
+
raise error if error.is_a?( SignalException )
|
71
|
+
end
|
72
|
+
end
|
73
|
+
elsif task.is_a?(Array) &&
|
74
|
+
task.first.is_a?(Boxxspring::ForbiddenError)
|
75
|
+
self.logger.error(task.first.message)
|
76
|
+
else
|
77
|
+
self.logger.error(
|
78
|
+
"The #{self.human_name} worker is unable to retrieve the " +
|
79
|
+
"task with the id #{task_id}."
|
80
|
+
)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
result
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
protected; def task_read( property_id, task_id )
|
91
|
+
# why did this not work?
|
92
|
+
# self.task_operation( property_id ).where( id: task_id ).read
|
93
|
+
Boxxspring::Operation.new(
|
94
|
+
"/properties/#{property_id}/tasks/#{task_id}",
|
95
|
+
Worker.configuration.api_credentials.to_hash
|
96
|
+
).read
|
97
|
+
end
|
98
|
+
|
99
|
+
protected; def task_write( task )
|
100
|
+
self.task_operation( task.property_id ).write( 'tasks', task ).first
|
101
|
+
end
|
102
|
+
|
103
|
+
protected; def task_write_state( task, state, message )
|
104
|
+
self.logger.send( ( state == 'failed' ? 'error' : 'info' ), message ) \
|
105
|
+
unless message.blank?
|
106
|
+
task.state = state
|
107
|
+
task.message = message
|
108
|
+
self.task_write( task )
|
109
|
+
end
|
110
|
+
|
111
|
+
protected; def task_operation( property_id )
|
112
|
+
Boxxspring::Operation.new(
|
113
|
+
"/properties/#{ property_id }/tasks",
|
114
|
+
Worker.configuration.api_credentials.to_hash
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
protected; def task_property_read( task, include = nil )
|
119
|
+
operation = Boxxspring::Operation.new(
|
120
|
+
"/properties/#{ task.property_id }",
|
121
|
+
Worker.configuration.api_credentials.to_hash
|
122
|
+
)
|
123
|
+
operation = operation.include( include ) \
|
124
|
+
unless ( include.blank? )
|
125
|
+
operation.read
|
126
|
+
end
|
127
|
+
|
128
|
+
protected; def task_delegate( queue_name, task )
|
129
|
+
serializer = Boxxspring::Serializer.new( task )
|
130
|
+
payload = serializer.serialize( 'tasks' )
|
131
|
+
payload.merge!( {
|
132
|
+
'$this' => {
|
133
|
+
'type_name' => 'tasks',
|
134
|
+
'unlimited_count' => 1
|
135
|
+
}
|
136
|
+
} )
|
137
|
+
self.delegate_payload( queue_name, payload )
|
138
|
+
end
|
139
|
+
|
140
|
+
protected; def operation(endpoint)
|
141
|
+
Boxxspring::Operation.new(
|
142
|
+
endpoint,
|
143
|
+
Boxxspring::Worker.configuration.api_credentials.to_hash
|
144
|
+
)
|
145
|
+
end
|
146
|
+
|
147
|
+
end
|
148
|
+
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift( File.expand_path( '..', File.dirname( __FILE__ ) ) )
|
5
|
+
require 'lib/boxxspring/abstract'
|
6
|
+
require 'lib/boxxspring/journal'
|
7
|
+
require 'lib/boxxspring/worker/configuration'
|
8
|
+
require 'lib/boxxspring/worker/base'
|
9
|
+
require 'lib/boxxspring/worker/task_base'
|
10
|
+
require 'lib/boxxspring/synchronization/configuration'
|
11
|
+
require 'lib/boxxspring/synchronization'
|
12
|
+
require 'lib/boxxspring/synchronization/operations'
|
13
|
+
require 'lib/boxxspring/synchronization/orchestrator'
|
14
|
+
require 'lib/boxxspring/synchronization/mutex'
|
15
|
+
require 'lib/boxxspring/synchronization/variable'
|
@@ -0,0 +1,38 @@
|
|
1
|
+
namespace :worker do
|
2
|
+
|
3
|
+
descendants = Boxxspring::Worker::Base.descendants
|
4
|
+
|
5
|
+
# remove base class workers
|
6
|
+
descendants.delete( Boxxspring::Worker::TaskBase )
|
7
|
+
|
8
|
+
descendants.each do | worker_class |
|
9
|
+
|
10
|
+
worker_name = worker_class.
|
11
|
+
name.
|
12
|
+
underscore.
|
13
|
+
gsub( /[\/]/, '-' ).
|
14
|
+
gsub( /_worker\Z/, '' )
|
15
|
+
|
16
|
+
desc "#{worker_name.humanize.downcase} worker."
|
17
|
+
task worker_name.to_sym do
|
18
|
+
spinner = %w{| / - \\}
|
19
|
+
worker = worker_class.new
|
20
|
+
print 'working... '
|
21
|
+
Boxxspring::Worker.configuration.logger.info(
|
22
|
+
"The #{worker_name.humanize.downcase} worker has started."
|
23
|
+
)
|
24
|
+
begin
|
25
|
+
loop do
|
26
|
+
print "\b" + spinner.rotate!.first
|
27
|
+
worker.process
|
28
|
+
end
|
29
|
+
rescue SystemExit, Interrupt
|
30
|
+
Boxxspring::Worker.configuration.logger.info(
|
31
|
+
"The #{worker_name.humanize.downcase} worker has stopped."
|
32
|
+
)
|
33
|
+
puts 'stopped'
|
34
|
+
exit 130
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
metadata
ADDED
@@ -0,0 +1,142 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: boxxspring-workers
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.1.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kristoph Cichocki-Romanov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: boxxspring
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: redis
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 3.2.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 3.2.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: remote_syslog_logger
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.10'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.10'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry-byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.1'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.1'
|
97
|
+
description: The boxxspring workers gem is implements the framework used to construct
|
98
|
+
boxxspring workers.
|
99
|
+
email: kristoph@bedrocket.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- lib/boxxspring-worker-version.rb
|
105
|
+
- lib/boxxspring-worker.rb
|
106
|
+
- lib/boxxspring/abstract.rb
|
107
|
+
- lib/boxxspring/journal.rb
|
108
|
+
- lib/boxxspring/synchronization.rb
|
109
|
+
- lib/boxxspring/synchronization/configuration.rb
|
110
|
+
- lib/boxxspring/synchronization/mutex.rb
|
111
|
+
- lib/boxxspring/synchronization/operations.rb
|
112
|
+
- lib/boxxspring/synchronization/orchestrator.rb
|
113
|
+
- lib/boxxspring/synchronization/variable.rb
|
114
|
+
- lib/boxxspring/worker/base.rb
|
115
|
+
- lib/boxxspring/worker/configuration.rb
|
116
|
+
- lib/boxxspring/worker/task_base.rb
|
117
|
+
- lib/tasks/workers.rake
|
118
|
+
homepage: http://bedrocket.com
|
119
|
+
licenses:
|
120
|
+
- MS-RL
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.2.2
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Bedrocket Media Ventrures Boxxspring Worker framework.
|
142
|
+
test_files: []
|