boxxspring-workers 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/boxxspring/abstract.rb +37 -0
- data/lib/boxxspring/journal.rb +60 -0
- data/lib/boxxspring/synchronization/configuration.rb +32 -0
- data/lib/boxxspring/synchronization/mutex.rb +24 -0
- data/lib/boxxspring/synchronization/operations.rb +81 -0
- data/lib/boxxspring/synchronization/orchestrator.rb +74 -0
- data/lib/boxxspring/synchronization/variable.rb +26 -0
- data/lib/boxxspring/synchronization.rb +17 -0
- data/lib/boxxspring/worker/base.rb +185 -0
- data/lib/boxxspring/worker/configuration.rb +41 -0
- data/lib/boxxspring/worker/task_base.rb +151 -0
- data/lib/boxxspring-worker-version.rb +5 -0
- data/lib/boxxspring-worker.rb +15 -0
- data/lib/tasks/workers.rake +38 -0
- metadata +142 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5a8ab0b25057afe43c90af53902ec1410d5230c2
|
4
|
+
data.tar.gz: f57b1b142e88bac02e52d893f1824fd63e19cacf
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 141d7d76f6209efe1b7beb56673094e1ed549dbf572dfef923524b4ab295ea35741b6961e4b655a612b91b0f52dabd8f81c47674fc74ca565b3f1c884e7d0e0a
|
7
|
+
data.tar.gz: 3f135b6596edf3a741e9df12d2870a9eafa8a3aded85205c198e17e94a2f651056452de89213be800bf81a54ed5bcfbb0ab38df2cb665786d91cb847b3a43abc
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
class Abstract
|
4
|
+
|
5
|
+
def initialize( attributes = {} )
|
6
|
+
@attributes = {}
|
7
|
+
attributes.each_pair do | key, value |
|
8
|
+
self.send( key, value )
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def to_hash
|
13
|
+
return @attributes
|
14
|
+
end
|
15
|
+
|
16
|
+
def method_missing( method, *arguments, &block )
|
17
|
+
result = nil
|
18
|
+
if arguments.length == 0
|
19
|
+
result = @attributes[ method.to_sym ]
|
20
|
+
if result.nil?
|
21
|
+
result = Abstract.new
|
22
|
+
result.instance_eval( &block ) unless block.nil?
|
23
|
+
@attributes[ method.to_sym ] = result
|
24
|
+
end
|
25
|
+
elsif arguments.length == 1
|
26
|
+
method = method.to_s.gsub( /=$/, '' )
|
27
|
+
result = arguments[ 0 ]
|
28
|
+
@attributes[ method.to_sym ] = result
|
29
|
+
else
|
30
|
+
result = super
|
31
|
+
end
|
32
|
+
result
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
class Journal
|
4
|
+
|
5
|
+
def initialize( name )
|
6
|
+
@db_name = name
|
7
|
+
end
|
8
|
+
|
9
|
+
def write( id, attributes )
|
10
|
+
db_attributes = []
|
11
|
+
attributes.each_pair do | key, value |
|
12
|
+
db_attributes.push( {
|
13
|
+
name: key.to_s, value: value.to_s, replace: true
|
14
|
+
} )
|
15
|
+
end
|
16
|
+
self.db.put_attributes( {
|
17
|
+
domain_name: @db_name,
|
18
|
+
item_name: id.to_s,
|
19
|
+
attributes: db_attributes
|
20
|
+
} )
|
21
|
+
end
|
22
|
+
|
23
|
+
def read( id )
|
24
|
+
result = nil
|
25
|
+
db_record = self.db.get_attributes(
|
26
|
+
domain_name: @db_name,
|
27
|
+
item_name: id.to_s,
|
28
|
+
consistent_read: true
|
29
|
+
)
|
30
|
+
if db_record.present? &&
|
31
|
+
db_record.attributes.present? &&
|
32
|
+
db_record.attributes.length > 0
|
33
|
+
result = {}
|
34
|
+
db_record.attributes.each do | attribute |
|
35
|
+
result[ attribute.name.to_sym ] = attribute.value
|
36
|
+
end
|
37
|
+
end
|
38
|
+
result
|
39
|
+
end
|
40
|
+
|
41
|
+
def delete( id )
|
42
|
+
db_record = self.db.delete_attributes(
|
43
|
+
domain_name: @db_name,
|
44
|
+
item_name: id.to_s
|
45
|
+
)
|
46
|
+
end
|
47
|
+
|
48
|
+
protected; def db
|
49
|
+
@db ||= begin
|
50
|
+
db = Aws::SimpleDB::Client.new
|
51
|
+
db.create_domain(
|
52
|
+
domain_name: @db_name
|
53
|
+
)
|
54
|
+
db
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Synchronization
|
4
|
+
|
5
|
+
class Configuration
|
6
|
+
|
7
|
+
include Singleton
|
8
|
+
|
9
|
+
def self.field( field_name, options = {} )
|
10
|
+
|
11
|
+
class_eval(
|
12
|
+
"def #{field_name}( *arguments ); " +
|
13
|
+
"@#{field_name} = arguments.first unless arguments.empty?; " +
|
14
|
+
"@#{field_name} || " +
|
15
|
+
( options[ :default ].blank? ?
|
16
|
+
"nil" :
|
17
|
+
( options[ :default ].is_a?( String ) ?
|
18
|
+
"'#{options[ :default ]}'" :
|
19
|
+
"#{options[ :default ]}" ) ) + ";" +
|
20
|
+
"end",
|
21
|
+
__FILE__,
|
22
|
+
__LINE__
|
23
|
+
)
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
field :url, default: "redis://127.0.0.1:6379/0"
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
class Mutex
|
5
|
+
|
6
|
+
def initialize( name, signature = nil )
|
7
|
+
@orchestrator = Synchronization::Orchestrator.instance
|
8
|
+
@name = name
|
9
|
+
@signature = signature || SecureRandom.hex
|
10
|
+
end
|
11
|
+
|
12
|
+
def lock( options = {} )
|
13
|
+
@orchestrator.lock( @name, @signature, options )
|
14
|
+
end
|
15
|
+
|
16
|
+
def unlock
|
17
|
+
@orchestrator.unlock( @name, @signature )
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
self.const_set 'Operations', {
|
5
|
+
|
6
|
+
lock:
|
7
|
+
"local result = false; "\
|
8
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
9
|
+
"if ( not value ) then "\
|
10
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
11
|
+
"result = true; "\
|
12
|
+
"elseif value == ARGV[ 1 ] then "\
|
13
|
+
"result = true; "\
|
14
|
+
"end; "\
|
15
|
+
"if result == true and not ( ARGV[ 2 ] == '' ) then "\
|
16
|
+
"redis.call( 'pexpire', KEYS[ 1 ], ARGV[ 2 ] ); "\
|
17
|
+
"end; "\
|
18
|
+
"return result",
|
19
|
+
|
20
|
+
unlock:
|
21
|
+
"local result = false; "\
|
22
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
23
|
+
"if ( not value ) then "\
|
24
|
+
"result = true; "\
|
25
|
+
"elseif value == ARGV[ 1 ] then "\
|
26
|
+
"redis.call( 'del', KEYS[ 1 ] ); "\
|
27
|
+
"result = true; "\
|
28
|
+
"else "\
|
29
|
+
"result = false; "\
|
30
|
+
"end; " \
|
31
|
+
"return result",
|
32
|
+
|
33
|
+
write_if_greater_than:
|
34
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
35
|
+
"if ( not value ) or ( ARGV[ 1 ] > value ) then "\
|
36
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
37
|
+
"return true; "\
|
38
|
+
"else "\
|
39
|
+
"return false;"\
|
40
|
+
"end",
|
41
|
+
|
42
|
+
write_if_greater_than_or_equal_to:
|
43
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
44
|
+
"if ( not value ) or ( ARGV[ 1 ] >= value ) then "\
|
45
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
46
|
+
"return true; "\
|
47
|
+
"else "\
|
48
|
+
"return false;"\
|
49
|
+
"end",
|
50
|
+
|
51
|
+
write_if_equal_to:
|
52
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
53
|
+
"if ( not value ) or ( ARGV[ 1 ] == value ) then "\
|
54
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
55
|
+
"return true; "\
|
56
|
+
"else "\
|
57
|
+
"return false;"\
|
58
|
+
"end",
|
59
|
+
|
60
|
+
write_if_less_than_or_equal_to:
|
61
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
62
|
+
"if ( not value ) or ( ARGV[ 1 ] <= value ) then "\
|
63
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
64
|
+
"return true; "\
|
65
|
+
"else "\
|
66
|
+
"return false;"\
|
67
|
+
"end",
|
68
|
+
|
69
|
+
write_if_less_than:
|
70
|
+
"local value = redis.call( 'get', KEYS[ 1 ] ); "\
|
71
|
+
"if ( not value ) or ( ARGV[ 1 ] < value ) then "\
|
72
|
+
"redis.call( 'set', KEYS[ 1 ], ARGV[ 1 ] ); "\
|
73
|
+
"return true; "\
|
74
|
+
"else "\
|
75
|
+
"return false;"\
|
76
|
+
"end"
|
77
|
+
|
78
|
+
}
|
79
|
+
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'redis'
|
2
|
+
|
3
|
+
module Boxxspring
|
4
|
+
module Synchronization
|
5
|
+
|
6
|
+
class Orchestrator
|
7
|
+
|
8
|
+
include Singleton
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@provider = ::Redis.new(
|
12
|
+
url: Synchronization.configuration.url,
|
13
|
+
timeout: 10.0
|
14
|
+
)
|
15
|
+
@operations = {}
|
16
|
+
end
|
17
|
+
|
18
|
+
def lock( key, signature, options = {} )
|
19
|
+
ttl = options[ :ttl ];
|
20
|
+
ttl = ttl * 1000 if ttl.is_a? ActiveSupport::Duration
|
21
|
+
self.execute_operation( :lock, [ key ], [ signature, ttl ] ) ?
|
22
|
+
true : false
|
23
|
+
end
|
24
|
+
|
25
|
+
def unlock( key, signature )
|
26
|
+
self.execute_operation( :unlock, [ key ], [ signature ] ) ?
|
27
|
+
true : false
|
28
|
+
end
|
29
|
+
|
30
|
+
def read( key )
|
31
|
+
@provider.get( key )
|
32
|
+
end
|
33
|
+
|
34
|
+
def write( key, value, options = {} )
|
35
|
+
ttl = options[ :ttl ]
|
36
|
+
if ttl
|
37
|
+
ttl = ttl.to_i if ttl.is_a? ActiveSupport::Duration
|
38
|
+
( @provider.set( key, value ) == "OK" ) &&
|
39
|
+
( @provider.expire( key, ttl ) ) ? true : false
|
40
|
+
else
|
41
|
+
( @provider.set( key, value ) == "OK" ) ? true : false
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def write_if_condition( key, value, condition )
|
46
|
+
operation = "write_if_#{condition}"
|
47
|
+
operation_sha = @operations[ operation.to_sym ]
|
48
|
+
raise 'Synchronization: An unknown condition was requested.' \
|
49
|
+
if operation_sha.nil?
|
50
|
+
@provider.evalsha( operation_sha, [ key ], [ value ] ) ? true : false
|
51
|
+
end
|
52
|
+
|
53
|
+
protected; def execute_operation( operation, keys, arguments )
|
54
|
+
try ||= 1
|
55
|
+
sha = @operations[ operation ] || install_operation( operation )
|
56
|
+
@provider.evalsha( sha, keys, arguments ) ? true : false
|
57
|
+
rescue Redis::CommandError => error
|
58
|
+
raise error unless error.message.match( /\ANOSCRIPT/ )
|
59
|
+
self.install_operation( operation )
|
60
|
+
retry if ( try -= 1 ) >= 0
|
61
|
+
end
|
62
|
+
|
63
|
+
protected; def install_operation( operation )
|
64
|
+
@operations[ operation ] = @provider.script(
|
65
|
+
:load,
|
66
|
+
Operations[ operation ]
|
67
|
+
)
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
module Synchronization
|
3
|
+
|
4
|
+
class Variable
|
5
|
+
|
6
|
+
def initialize( name )
|
7
|
+
@orchestrator = Synchronization::Orchestrator.instance
|
8
|
+
@name = name
|
9
|
+
end
|
10
|
+
|
11
|
+
def read
|
12
|
+
@orchestrator.read( @name )
|
13
|
+
end
|
14
|
+
|
15
|
+
def write( value, options={} )
|
16
|
+
@orchestrator.write( @name, value, options )
|
17
|
+
end
|
18
|
+
|
19
|
+
def write_if_condition( value, condition )
|
20
|
+
@orchestrator.write_if_condition( @name, value, condition )
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
#
|
2
|
+
# module. Synchronization
|
3
|
+
#
|
4
|
+
# The synchornization module implements lock/unlock and key/value read/write
|
5
|
+
# operations accross multiple instances of the application.
|
6
|
+
|
7
|
+
module Boxxspring
|
8
|
+
module Synchronization
|
9
|
+
|
10
|
+
def self.configuration( &block )
|
11
|
+
configuration = Synchronization::Configuration.instance
|
12
|
+
configuration.instance_eval( &block ) unless block.nil?
|
13
|
+
configuration
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,185 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Worker
|
4
|
+
|
5
|
+
QUEUE_MESSAGE_REQUEST_COUNT = 10
|
6
|
+
QUEUE_MESSAGE_WAIT_IN_SECONDS = 4
|
7
|
+
|
8
|
+
class Base
|
9
|
+
|
10
|
+
#------------------------------------------------------------------------
|
11
|
+
# class attributes
|
12
|
+
|
13
|
+
class_attribute :queue_name
|
14
|
+
class_attribute :processor
|
15
|
+
|
16
|
+
#------------------------------------------------------------------------
|
17
|
+
# class methods
|
18
|
+
|
19
|
+
class << self
|
20
|
+
|
21
|
+
public; def process( &block )
|
22
|
+
self.processor = block
|
23
|
+
end
|
24
|
+
|
25
|
+
def queue_interface
|
26
|
+
@queue_interface ||= Aws::SQS::Client.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def queue_url
|
30
|
+
unless @queue_url.present?
|
31
|
+
response = self.queue_interface.create_queue(
|
32
|
+
queue_name: self.full_queue_name
|
33
|
+
)
|
34
|
+
@queue_url = response[ :queue_url ]
|
35
|
+
end
|
36
|
+
@queue_url
|
37
|
+
end
|
38
|
+
|
39
|
+
protected; def full_queue_name
|
40
|
+
name = ( Worker.env == 'development' ) ?
|
41
|
+
( ENV[ 'USER' ] || 'development' ) :
|
42
|
+
Worker.env
|
43
|
+
name += '-' +
|
44
|
+
( self.queue_name ||
|
45
|
+
self.name.
|
46
|
+
underscore.
|
47
|
+
gsub( /[\/]/, '-' ).
|
48
|
+
gsub( /_worker\Z/, '' ) )
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
#------------------------------------------------------------------------
|
54
|
+
# operations
|
55
|
+
|
56
|
+
def process
|
57
|
+
messages = self.receive_messages() || []
|
58
|
+
messages.each do | message |
|
59
|
+
if message.present?
|
60
|
+
payload = self.payload_from_message( message )
|
61
|
+
if payload.present?
|
62
|
+
begin
|
63
|
+
result = self.process_payload( payload )
|
64
|
+
# note: if an exception is raised the message will be deleted
|
65
|
+
self.delete_message( message ) unless result == false
|
66
|
+
rescue StandardError => error
|
67
|
+
self.logger.error(
|
68
|
+
"The #{ self.human_name } worker failed to process the " +
|
69
|
+
"payload. "
|
70
|
+
)
|
71
|
+
self.logger.error( error.message )
|
72
|
+
self.logger.error( error.backtrace.join( "\n" ) )
|
73
|
+
end
|
74
|
+
else
|
75
|
+
# note: messages with invalid payloads are deleted
|
76
|
+
self.delete_message( message )
|
77
|
+
self.logger.error(
|
78
|
+
"The #{ self.human_name } worker received an invalid payload."
|
79
|
+
)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
protected; def logger
|
86
|
+
@logger ||= Boxxspring::Worker.configuration.logger
|
87
|
+
end
|
88
|
+
|
89
|
+
#------------------------------------------------------------------------
|
90
|
+
# implementation
|
91
|
+
|
92
|
+
protected; def receive_messages
|
93
|
+
messages = nil
|
94
|
+
begin
|
95
|
+
response = self.class.queue_interface.receive_message(
|
96
|
+
queue_url: self.class.queue_url,
|
97
|
+
max_number_of_messages: QUEUE_MESSAGE_REQUEST_COUNT,
|
98
|
+
wait_time_seconds: QUEUE_MESSAGE_WAIT_IN_SECONDS
|
99
|
+
)
|
100
|
+
messages = response[ :messages ]
|
101
|
+
rescue StandardError => error
|
102
|
+
raise RuntimeError.new(
|
103
|
+
"The #{ self.human_name } worker is unable to receive a message " +
|
104
|
+
"from the queue. #{error.message}."
|
105
|
+
)
|
106
|
+
end
|
107
|
+
messages
|
108
|
+
end
|
109
|
+
|
110
|
+
protected; def delete_message( message )
|
111
|
+
begin
|
112
|
+
self.class.queue_interface.delete_message(
|
113
|
+
queue_url: self.class.queue_url,
|
114
|
+
receipt_handle: message[ :receipt_handle ]
|
115
|
+
)
|
116
|
+
rescue StandardError => error
|
117
|
+
raise RuntimeError.new(
|
118
|
+
"The #{ self.human_name } worker is unable to delete the " +
|
119
|
+
"message from the queue. #{error.message}."
|
120
|
+
)
|
121
|
+
end
|
122
|
+
message
|
123
|
+
end
|
124
|
+
|
125
|
+
protected; def payload_from_message( message )
|
126
|
+
payload = message.body
|
127
|
+
if ( payload.present? )
|
128
|
+
payload = JSON.parse( payload ) rescue payload
|
129
|
+
if ( payload.is_a?( Hash ) &&
|
130
|
+
payload.include?( 'Type' ) &&
|
131
|
+
payload[ 'Type' ] == 'Notification' )
|
132
|
+
payload = payload[ 'Message' ]
|
133
|
+
payload = payload.present? ?
|
134
|
+
( JSON.parse( payload ) rescue payload ) :
|
135
|
+
payload
|
136
|
+
end
|
137
|
+
end
|
138
|
+
payload
|
139
|
+
end
|
140
|
+
|
141
|
+
protected; def process_payload( payload )
|
142
|
+
if self.class.processor.present?
|
143
|
+
self.class.processor.call( payload )
|
144
|
+
else
|
145
|
+
raise RuntimeError.new(
|
146
|
+
"The worker lacks a processor"
|
147
|
+
)
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
protected; def delegate_payload( queue_name, payload )
|
152
|
+
queue_name_prefix = ( Worker.env == 'development' ) ?
|
153
|
+
( ENV[ 'USER' ] || 'development' ) : Worker.env
|
154
|
+
queue_name = queue_name_prefix + '-' + queue_name
|
155
|
+
begin
|
156
|
+
response = self.class.queue_interface.create_queue(
|
157
|
+
queue_name: queue_name
|
158
|
+
)
|
159
|
+
queue_url = response[ :queue_url ]
|
160
|
+
if queue_url.present?
|
161
|
+
self.class.queue_interface.send_message(
|
162
|
+
queue_url: queue_url,
|
163
|
+
message_body: payload.to_json
|
164
|
+
)
|
165
|
+
end
|
166
|
+
rescue StandardError => error
|
167
|
+
raise RuntimeError.new(
|
168
|
+
"The #{ self.human_name } worker was unable to delegate the " +
|
169
|
+
"payload to the queue name '#{ queue_name }'. #{error.message}."
|
170
|
+
)
|
171
|
+
end
|
172
|
+
end
|
173
|
+
|
174
|
+
protected; def human_name
|
175
|
+
self.class.name.
|
176
|
+
underscore.
|
177
|
+
gsub( /[\/]/, ' ' ).
|
178
|
+
gsub( /_worker\Z/, '' )
|
179
|
+
end
|
180
|
+
|
181
|
+
end
|
182
|
+
|
183
|
+
end
|
184
|
+
|
185
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Boxxspring
|
4
|
+
|
5
|
+
module Worker
|
6
|
+
|
7
|
+
def self.configuration( &block )
|
8
|
+
Configuration.instance().instance_eval( &block ) unless block.nil?
|
9
|
+
Configuration.instance()
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.env
|
13
|
+
self.configuration.env
|
14
|
+
end
|
15
|
+
|
16
|
+
class Configuration < Abstract
|
17
|
+
|
18
|
+
include Singleton
|
19
|
+
|
20
|
+
def initialize
|
21
|
+
super( {
|
22
|
+
env: ENV[ 'WORKERS_ENV' ] || 'development',
|
23
|
+
logger: Logger.new( STDOUT )
|
24
|
+
} )
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.reloadable?
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
def from_hash( configuration )
|
32
|
+
configuration.each_pair do | name, value |
|
33
|
+
self.send( "@#{name}", value )
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
module Boxxspring
|
2
|
+
|
3
|
+
module Worker
|
4
|
+
|
5
|
+
class TaskBase < Base
|
6
|
+
|
7
|
+
#------------------------------------------------------------------------
|
8
|
+
# class attributes
|
9
|
+
|
10
|
+
class_attribute :task_type_name
|
11
|
+
class_attribute :task_state
|
12
|
+
|
13
|
+
#------------------------------------------------------------------------
|
14
|
+
# operations
|
15
|
+
|
16
|
+
protected; def process_task( task )
|
17
|
+
if self.class.processor.present?
|
18
|
+
self.class.processor.call( task )
|
19
|
+
else
|
20
|
+
raise RuntimeError.new(
|
21
|
+
"The #{self.human_name} worker lacks a task processor"
|
22
|
+
)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
#------------------------------------------------------------------------
|
27
|
+
# implementation
|
28
|
+
|
29
|
+
protected; def process_payload( payload )
|
30
|
+
|
31
|
+
result = true
|
32
|
+
type_names = self.task_type_name.blank? ?
|
33
|
+
nil :
|
34
|
+
[ self.task_type_name ].flatten
|
35
|
+
states = self.task_state.blank? ?
|
36
|
+
nil :
|
37
|
+
[ self.task_state ].flatten
|
38
|
+
|
39
|
+
tasks = payload[ 'tasks' ]
|
40
|
+
if ( tasks.present? && tasks.respond_to?( :each ) )
|
41
|
+
tasks.each do | task |
|
42
|
+
task_id = task[ 'id' ]
|
43
|
+
if ( type_names.blank? || type_names.include?( task[ 'type_name' ] ) )
|
44
|
+
task = task_read( task[ 'property_id' ], task_id )
|
45
|
+
if task.is_a?( Boxxspring::Task )
|
46
|
+
if ( states.blank? || states.include?( task.state ) )
|
47
|
+
self.logger.info(
|
48
|
+
"The task (id: #{task.id}) processing has started."
|
49
|
+
)
|
50
|
+
begin
|
51
|
+
result = self.process_task( task )
|
52
|
+
message = "The task (id: #{task.id}) processing has ended"
|
53
|
+
message += " and the message has been retained." if result == false
|
54
|
+
self.logger.info(
|
55
|
+
message
|
56
|
+
)
|
57
|
+
rescue SignalException, StandardError => error
|
58
|
+
if error.is_a?( SignalException )
|
59
|
+
task_state = 'idle'
|
60
|
+
task_message = "The task (id: #{task.id}) has restarted."
|
61
|
+
else
|
62
|
+
task_state = 'failed'
|
63
|
+
task_message = "The task (id: #{task.id}) processing has failed."
|
64
|
+
end
|
65
|
+
task = task_write_state(task,
|
66
|
+
task_state,
|
67
|
+
task_message)
|
68
|
+
self.logger.error( error.message )
|
69
|
+
self.logger.error( error.backtrace.join( "\n" ) )
|
70
|
+
raise error if error.is_a?( SignalException )
|
71
|
+
end
|
72
|
+
end
|
73
|
+
elsif task.is_a?(Array) &&
|
74
|
+
task.first.is_a?(Boxxspring::ForbiddenError)
|
75
|
+
self.logger.error(task.first.message)
|
76
|
+
else
|
77
|
+
self.logger.error(
|
78
|
+
"The #{self.human_name} worker is unable to retrieve the " +
|
79
|
+
"task with the id #{task_id}."
|
80
|
+
)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
result
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
protected; def task_read( property_id, task_id )
|
91
|
+
# why did this not work?
|
92
|
+
# self.task_operation( property_id ).where( id: task_id ).read
|
93
|
+
Boxxspring::Operation.new(
|
94
|
+
"/properties/#{property_id}/tasks/#{task_id}",
|
95
|
+
Worker.configuration.api_credentials.to_hash
|
96
|
+
).read
|
97
|
+
end
|
98
|
+
|
99
|
+
protected; def task_write( task )
|
100
|
+
self.task_operation( task.property_id ).write( 'tasks', task ).first
|
101
|
+
end
|
102
|
+
|
103
|
+
protected; def task_write_state( task, state, message )
|
104
|
+
self.logger.send( ( state == 'failed' ? 'error' : 'info' ), message ) \
|
105
|
+
unless message.blank?
|
106
|
+
task.state = state
|
107
|
+
task.message = message
|
108
|
+
self.task_write( task )
|
109
|
+
end
|
110
|
+
|
111
|
+
protected; def task_operation( property_id )
|
112
|
+
Boxxspring::Operation.new(
|
113
|
+
"/properties/#{ property_id }/tasks",
|
114
|
+
Worker.configuration.api_credentials.to_hash
|
115
|
+
)
|
116
|
+
end
|
117
|
+
|
118
|
+
protected; def task_property_read( task, include = nil )
|
119
|
+
operation = Boxxspring::Operation.new(
|
120
|
+
"/properties/#{ task.property_id }",
|
121
|
+
Worker.configuration.api_credentials.to_hash
|
122
|
+
)
|
123
|
+
operation = operation.include( include ) \
|
124
|
+
unless ( include.blank? )
|
125
|
+
operation.read
|
126
|
+
end
|
127
|
+
|
128
|
+
protected; def task_delegate( queue_name, task )
|
129
|
+
serializer = Boxxspring::Serializer.new( task )
|
130
|
+
payload = serializer.serialize( 'tasks' )
|
131
|
+
payload.merge!( {
|
132
|
+
'$this' => {
|
133
|
+
'type_name' => 'tasks',
|
134
|
+
'unlimited_count' => 1
|
135
|
+
}
|
136
|
+
} )
|
137
|
+
self.delegate_payload( queue_name, payload )
|
138
|
+
end
|
139
|
+
|
140
|
+
protected; def operation(endpoint)
|
141
|
+
Boxxspring::Operation.new(
|
142
|
+
endpoint,
|
143
|
+
Boxxspring::Worker.configuration.api_credentials.to_hash
|
144
|
+
)
|
145
|
+
end
|
146
|
+
|
147
|
+
end
|
148
|
+
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
require 'active_support/all'
|
3
|
+
|
4
|
+
$LOAD_PATH.unshift( File.expand_path( '..', File.dirname( __FILE__ ) ) )
|
5
|
+
require 'lib/boxxspring/abstract'
|
6
|
+
require 'lib/boxxspring/journal'
|
7
|
+
require 'lib/boxxspring/worker/configuration'
|
8
|
+
require 'lib/boxxspring/worker/base'
|
9
|
+
require 'lib/boxxspring/worker/task_base'
|
10
|
+
require 'lib/boxxspring/synchronization/configuration'
|
11
|
+
require 'lib/boxxspring/synchronization'
|
12
|
+
require 'lib/boxxspring/synchronization/operations'
|
13
|
+
require 'lib/boxxspring/synchronization/orchestrator'
|
14
|
+
require 'lib/boxxspring/synchronization/mutex'
|
15
|
+
require 'lib/boxxspring/synchronization/variable'
|
@@ -0,0 +1,38 @@
|
|
1
|
+
namespace :worker do
|
2
|
+
|
3
|
+
descendants = Boxxspring::Worker::Base.descendants
|
4
|
+
|
5
|
+
# remove base class workers
|
6
|
+
descendants.delete( Boxxspring::Worker::TaskBase )
|
7
|
+
|
8
|
+
descendants.each do | worker_class |
|
9
|
+
|
10
|
+
worker_name = worker_class.
|
11
|
+
name.
|
12
|
+
underscore.
|
13
|
+
gsub( /[\/]/, '-' ).
|
14
|
+
gsub( /_worker\Z/, '' )
|
15
|
+
|
16
|
+
desc "#{worker_name.humanize.downcase} worker."
|
17
|
+
task worker_name.to_sym do
|
18
|
+
spinner = %w{| / - \\}
|
19
|
+
worker = worker_class.new
|
20
|
+
print 'working... '
|
21
|
+
Boxxspring::Worker.configuration.logger.info(
|
22
|
+
"The #{worker_name.humanize.downcase} worker has started."
|
23
|
+
)
|
24
|
+
begin
|
25
|
+
loop do
|
26
|
+
print "\b" + spinner.rotate!.first
|
27
|
+
worker.process
|
28
|
+
end
|
29
|
+
rescue SystemExit, Interrupt
|
30
|
+
Boxxspring::Worker.configuration.logger.info(
|
31
|
+
"The #{worker_name.humanize.downcase} worker has stopped."
|
32
|
+
)
|
33
|
+
puts 'stopped'
|
34
|
+
exit 130
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
metadata
ADDED
@@ -0,0 +1,142 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: boxxspring-workers
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.1.4
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Kristoph Cichocki-Romanov
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-15 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: aws-sdk
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: boxxspring
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: redis
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 3.2.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 3.2.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: remote_syslog_logger
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: pry
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.10'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.10'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry-byebug
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.1'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.1'
|
97
|
+
description: The boxxspring workers gem is implements the framework used to construct
|
98
|
+
boxxspring workers.
|
99
|
+
email: kristoph@bedrocket.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- lib/boxxspring-worker-version.rb
|
105
|
+
- lib/boxxspring-worker.rb
|
106
|
+
- lib/boxxspring/abstract.rb
|
107
|
+
- lib/boxxspring/journal.rb
|
108
|
+
- lib/boxxspring/synchronization.rb
|
109
|
+
- lib/boxxspring/synchronization/configuration.rb
|
110
|
+
- lib/boxxspring/synchronization/mutex.rb
|
111
|
+
- lib/boxxspring/synchronization/operations.rb
|
112
|
+
- lib/boxxspring/synchronization/orchestrator.rb
|
113
|
+
- lib/boxxspring/synchronization/variable.rb
|
114
|
+
- lib/boxxspring/worker/base.rb
|
115
|
+
- lib/boxxspring/worker/configuration.rb
|
116
|
+
- lib/boxxspring/worker/task_base.rb
|
117
|
+
- lib/tasks/workers.rake
|
118
|
+
homepage: http://bedrocket.com
|
119
|
+
licenses:
|
120
|
+
- MS-RL
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - ">="
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.2.2
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Bedrocket Media Ventrures Boxxspring Worker framework.
|
142
|
+
test_files: []
|