shoryuken-later 0.0.6 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4ed2b69dae97667b38ef15a0be1ee0f6989e9c95
4
- data.tar.gz: 82fa4cb2cb2f763be48094df3f4224a025d56815
3
+ metadata.gz: c0ffbca529e8f0d07572a00983be2e0c708044f1
4
+ data.tar.gz: 65529758322a4bcdb4c38d18fe66f8f7c3f9e59c
5
5
  SHA512:
6
- metadata.gz: 9f1d6efd2b3521a763e685e65d3c695fa8282ab6f3b0b4b0dc6450d26a0b656ffffaf40057db9bf35cee63ac1f9708715649581ff0094ddf2d0bd707a195cf02
7
- data.tar.gz: f5f07fd722f05fc37206b8e4a59e1acf2960282a7656d60275df746458b60bbf0b3be0e2e82d649fd5ae4fefc0875025cfd84d4e839bb45927c20f38453d706a
6
+ metadata.gz: c64832d614bda4caab3b05b13293acab6a36f7c2352971a0ac64bcf358034547630425ceb5e534eb2176edde0e18b933e060687ca8a20cf2ca5f4306fe2e6955
7
+ data.tar.gz: a1a8e0ad1961ebc4ce371b26598c1c2ac624faac4ef2a2ae705fa7ccbf5aca5173b6d7e9a8f2955aa34dca937872f617c15b608e5df586da941d282556250cc7
@@ -30,12 +30,13 @@ module ActiveJob
30
30
 
31
31
  delay = (timestamp - Time.current.to_f).round
32
32
  if delay > 15.minutes
33
- Shoryuken::Later::Client.put_item(Shoryuken::Later.default_table, perform_at: Time.current.to_i + delay.to_i,
34
- shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
35
- shoryuken_args: JSON.dump(body: job.serialize, options: {}))
33
+ Shoryuken::Later::Client.create_item(Shoryuken::Later.default_table, perform_at: Time.current.to_i + delay.to_i,
34
+ shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
35
+ shoryuken_args: JSON.dump(body: job.serialize, options: {}))
36
36
  else
37
- Shoryuken::Client.send_message(job.queue_name, job.serialize, delay_seconds: delay,
38
- message_attributes: message_attributes)
37
+ Shoryuken::Client.queues(job.queue_name).send_message(message_body: job.serialize,
38
+ message_attributes: message_attributes,
39
+ delay_seconds: delay)
39
40
  end
40
41
  end
41
42
  end
@@ -18,7 +18,7 @@ module Shoryuken
18
18
  def run(args)
19
19
  self_read, self_write = IO.pipe
20
20
 
21
- %w[INT TERM USR1 USR2 TTIN].each do |sig|
21
+ %w[INT TERM USR1 USR2].each do |sig|
22
22
  trap sig do
23
23
  self_write.puts(sig)
24
24
  end
@@ -34,31 +34,33 @@ module Shoryuken
34
34
  daemonize
35
35
  write_pid
36
36
 
37
- logger.info 'Starting'
38
-
39
- # Initialize the timers and poller.
40
- @timers = Timers::Group.new
41
- require 'shoryuken/later/poller'
42
- @pollers = Shoryuken::Later.tables.map{|tbl| Poller.new(tbl) }
43
-
44
- begin
45
- # Poll for items on startup, and every :poll_delay
46
- poll_tables
47
- @timers.every(Shoryuken::Later.poll_delay){ poll_tables }
37
+ Shoryuken::Logging.with_context '[later]' do
38
+ logger.info 'Starting'
48
39
 
49
- # Loop watching for signals and firing off of timers
50
- loop do
51
- interval = @timers.wait_interval
52
- readable, writable = IO.select([self_read], nil, nil, interval)
53
- if readable
54
- handle_signal readable.first.gets.strip
55
- else
56
- @timers.fire
40
+ # Initialize the timers and poller.
41
+ @timers = Timers::Group.new
42
+ require 'shoryuken/later/poller'
43
+ @pollers = Shoryuken::Later.tables.map{|tbl| Poller.new(tbl) }
44
+
45
+ begin
46
+ # Poll for items on startup, and every :poll_delay
47
+ poll_tables
48
+ @timers.every(Shoryuken::Later.poll_delay){ poll_tables }
49
+
50
+ # Loop watching for signals and firing off of timers
51
+ while @timers
52
+ interval = @timers.wait_interval
53
+ readable, writable = IO.select([self_read], nil, nil, interval)
54
+ if readable
55
+ handle_signal readable.first.gets.strip
56
+ else
57
+ @timers.fire
58
+ end
57
59
  end
60
+ rescue Interrupt
61
+ @timers.cancel
62
+ exit 0
58
63
  end
59
- rescue Interrupt
60
- @timers.cancel
61
- exit 0
62
64
  end
63
65
  end
64
66
 
@@ -191,8 +193,7 @@ module Shoryuken
191
193
  when 'USR1'
192
194
  logger.info "Received USR1, will soft shutdown down"
193
195
  @timers.cancel
194
- sleep 1 while @busy
195
- exit 0
196
+ @timers = nil
196
197
  else
197
198
  logger.info "Received #{sig}, will shutdown down"
198
199
  raise Interrupt
@@ -252,11 +253,9 @@ module Shoryuken
252
253
  Shoryuken::Later.tables.uniq.each do |table|
253
254
  # validate all tables and AWS credentials consequently
254
255
  begin
255
- unless Shoryuken::Later::Client.tables(table).exists?
256
- raise ArgumentError, "Table '#{table}' does not exist"
257
- end
258
- rescue => e
259
- raise
256
+ Shoryuken::Later::Client.tables table
257
+ rescue Aws::DynamoDB::Errors::ResourceNotFoundException => e
258
+ raise ArgumentError, "Table '#{table}' does not exist"
260
259
  end
261
260
  end
262
261
  end
@@ -264,7 +263,23 @@ module Shoryuken
264
263
  def initialize_aws
265
264
  # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
266
265
  # when not explicit supplied
267
- AWS.config Shoryuken::Later.options[:aws] if Shoryuken::Later.options[:aws]
266
+ return if Shoryuken::Later.options[:aws].empty?
267
+
268
+ shoryuken_keys = %i(
269
+ account_id
270
+ sns_endpoint
271
+ sqs_endpoint
272
+ receive_message)
273
+
274
+ aws_options = Shoryuken::Later.options[:aws].reject do |k, v|
275
+ shoryuken_keys.include?(k)
276
+ end
277
+
278
+ credentials = Aws::Credentials.new(
279
+ aws_options.delete(:access_key_id),
280
+ aws_options.delete(:secret_access_key))
281
+
282
+ Aws.config = aws_options.merge(credentials: credentials)
268
283
  end
269
284
 
270
285
  def require_workers
@@ -4,21 +4,31 @@ module Shoryuken
4
4
  module Later
5
5
  class Client
6
6
  @@tables = {}
7
-
7
+
8
8
  class << self
9
9
  def tables(table)
10
- @@tables[table.to_s] ||= ddb.tables[table].tap do |tbl|
11
- tbl.hash_key = [:id, :string]
12
- end
10
+ @@tables[table] ||= ddb.describe_table(table_name: table)
11
+ end
12
+
13
+ def first_item(table, filter=nil)
14
+ result = ddb.scan(table_name: table, limit: 1, scan_filter: filter)
15
+ result.items.first if result
16
+ end
17
+
18
+ def create_item(table, item)
19
+ item['id'] ||= SecureRandom.uuid
20
+
21
+ ddb.put_item(table_name: table, item: item,
22
+ expected: {id: {exists: false}})
13
23
  end
14
24
 
15
- def put_item(table, attributes, options={unless_exists: 'id'})
16
- attributes['id'] ||= SecureRandom.uuid
17
- tables(table).items.create(attributes, options)
25
+ def delete_item(table, item)
26
+ ddb.delete_item(table_name: table, key: {id: item['id']},
27
+ expected: {id: {value: item['id'], exists: true}})
18
28
  end
19
29
 
20
30
  def ddb
21
- @ddb ||= AWS::DynamoDB.new
31
+ @ddb ||= Aws::DynamoDB::Client.new
22
32
  end
23
33
  end
24
34
  end
@@ -15,20 +15,20 @@ module Shoryuken
15
15
  watchdog('Later::Poller#poll died') do
16
16
  started_at = Time.now
17
17
 
18
- logger.debug { "Polling for scheduled messages in '#{@table_name}'" }
18
+ logger.debug { "Polling for scheduled messages in '#{table_name}'" }
19
19
 
20
20
  begin
21
21
  while item = next_item
22
- id = item.attributes['id']
23
- logger.info "Found message #{id} from '#{@table_name}'"
22
+ id = item['id']
23
+ logger.info "Found message #{id} from '#{table_name}'"
24
24
  if sent_msg = process_item(item)
25
- logger.debug { "Enqueued message #{id} from '#{@table_name}' as #{sent_msg.id}" }
25
+ logger.debug { "Enqueued message #{id} from '#{table_name}'" }
26
26
  else
27
- logger.debug { "Skipping already queued message #{id} from '#{@table_name}'" }
27
+ logger.debug { "Skipping already queued message #{id} from '#{table_name}'" }
28
28
  end
29
29
  end
30
30
 
31
- logger.debug { "Poller for '#{@table_name}' completed in #{elapsed(started_at)} ms" }
31
+ logger.debug { "Poller for '#{table_name}' completed in #{elapsed(started_at)} ms" }
32
32
  rescue => ex
33
33
  logger.error "Error fetching message: #{ex}"
34
34
  logger.error ex.backtrace.first
@@ -38,26 +38,29 @@ module Shoryuken
38
38
 
39
39
  private
40
40
 
41
- def table
42
- Shoryuken::Later::Client.tables(@table_name)
41
+ def client
42
+ Shoryuken::Later::Client
43
43
  end
44
-
44
+
45
45
  # Fetches the next available item from the schedule table.
46
46
  def next_item
47
- table.items.where(:perform_at).less_than((Time.now + Shoryuken::Later::MAX_QUEUE_DELAY).to_i).first
47
+ client.first_item table_name, 'perform_at' => {
48
+ attribute_value_list: [ (Time.now + Shoryuken::Later::MAX_QUEUE_DELAY).to_i ],
49
+ comparison_operator: 'LT'
50
+ }
48
51
  end
49
52
 
50
53
  # Processes an item and enqueues it (unless another actor has already enqueued it).
51
54
  def process_item(item)
52
- time, worker_class, args, id = item.attributes.values_at('perform_at','shoryuken_class','shoryuken_args','id')
55
+ time, worker_class, args, id = item.values_at('perform_at','shoryuken_class','shoryuken_args','id')
53
56
 
54
57
  worker_class = worker_class.constantize
55
58
  args = JSON.parse(args)
56
59
  time = Time.at(time)
57
- queue_name = item.attributes['shoryuken_queue']
60
+ queue_name = item['shoryuken_queue']
58
61
 
59
62
  # Conditionally delete an item prior to enqueuing it, ensuring only one actor may enqueue it.
60
- begin item.delete(:if => {id: id})
63
+ begin client.delete_item table_name, item
61
64
  rescue AWS::DynamoDB::Errors::ConditionalCheckFailedException => e
62
65
  # Item was already deleted, so it does not need to be queued.
63
66
  return
@@ -71,10 +74,12 @@ module Shoryuken
71
74
  # For compatibility with Shoryuken's ActiveJob adapter, support an explicit queue name.
72
75
  else
73
76
  delay = (time - Time.now).to_i
77
+ body = JSON.dump(body) if body.is_a? Hash
74
78
  options[:delay_seconds] = delay if delay > 0
79
+ options[:message_body] = body
75
80
  options[:message_attributes] ||= {}
76
81
  options[:message_attributes]['shoryuken_class'] = { string_value: worker_class.to_s, data_type: 'String' }
77
- Shoryuken::Client.send_message(queue_name, body, options)
82
+ Shoryuken::Client.queues(queue_name).send_message(options)
78
83
  end
79
84
  end
80
85
 
@@ -1,5 +1,5 @@
1
1
  module Shoryuken
2
2
  module Later
3
- VERSION = '0.0.6'
3
+ VERSION = '0.1.0'
4
4
  end
5
5
  end
@@ -19,8 +19,8 @@ module Shoryuken
19
19
  else
20
20
  table = get_shoryuken_options['schedule_table'] || Shoryuken::Later.default_table
21
21
  args = JSON.dump(body: body, options: options)
22
- Shoryuken::Later::Client.put_item(table, perform_at: time.to_i, shoryuken_args: args,
23
- shoryuken_class: self.to_s)
22
+ Shoryuken::Later::Client.create_item(table, perform_at: time.to_i, shoryuken_args: args,
23
+ shoryuken_class: self.to_s)
24
24
  end
25
25
  end
26
26
  end
@@ -27,7 +27,8 @@ Gem::Specification.new do |spec|
27
27
  spec.add_development_dependency "rspec", '~> 3.0', '< 3.1'
28
28
  spec.add_development_dependency "pry-byebug"
29
29
 
30
- spec.add_dependency "aws-sdk-v1"
31
30
  spec.add_dependency "timers", "~> 4.0.1"
32
- spec.add_dependency "shoryuken", "~> 0.0.5"
31
+ spec.add_dependency "aws-sdk-core", "2.0.21"
32
+ spec.add_dependency "aws-sdk-resources", "2.0.21.pre"
33
+ spec.add_dependency "shoryuken", "> 0.0.5"
33
34
  end
@@ -1,42 +1,36 @@
1
1
  require 'spec_helper'
2
2
 
3
3
  describe Shoryuken::Later::Client do
4
- let(:ddb) { double 'DynamoDB' }
5
- let(:table_collection) { double 'Table Collection' }
6
- let(:ddb_table) { double 'DynamoDb Table' }
7
- let(:ddb_items) { double 'Table Items' }
8
- let(:table) { 'shoryuken_later' }
4
+ let(:ddb) { double 'DynamoDB' }
5
+ let(:table_description) { double 'Table Description' }
6
+ let(:table) { 'shoryuken_later' }
9
7
 
10
8
  before do
11
9
  allow(described_class).to receive(:ddb).and_return(ddb)
12
- allow(ddb).to receive(:tables).and_return(table_collection)
13
- allow(table_collection).to receive(:[]).and_return(ddb_table)
14
- allow(ddb_table).to receive(:items).and_return(ddb_items)
15
- allow(ddb_table).to receive(:hash_key=)
10
+ allow(ddb).to receive(:describe_table).and_return(table_description)
16
11
  end
17
12
 
18
13
  describe '.tables' do
19
- it 'memoizes tables and sets the hash_key' do
20
- expect(table_collection).to receive(:[]).once.with(table).and_return(ddb_table)
21
- expect(ddb_table).to receive(:hash_key=).once.with([:id, :string])
14
+ it 'memoizes tables' do
15
+ expect(ddb).to receive(:describe_table).once.with(table_name: table).and_return(table_description)
22
16
 
23
- expect(described_class.tables(table)).to eq(ddb_table)
24
- expect(described_class.tables(table)).to eq(ddb_table)
17
+ expect(described_class.tables(table)).to eq(table_description)
18
+ expect(described_class.tables(table)).to eq(table_description)
25
19
  end
26
20
  end
27
21
 
28
- describe '.put_item' do
22
+ describe '.create_item' do
29
23
  it 'creates an item with a supplied ID' do
30
- expect(ddb_items).to receive(:create).with({'id' => 'fubar'}, {unless_exists: 'id'})
24
+ expect(ddb).to receive(:put_item).with(table_name: table, item: {'id' => 'fubar'}, expected: {id: {exists: false}})
31
25
 
32
- described_class.put_item(table,'id' => 'fubar')
26
+ described_class.create_item(table,'id' => 'fubar')
33
27
  end
34
28
 
35
29
  it 'creates an item with a auto-generated ID' do
36
30
  expect(SecureRandom).to receive(:uuid).once.and_return('fubar')
37
- expect(ddb_items).to receive(:create).with({'id' => 'fubar', 'perform_at' => 1234}, {unless_exists: 'id'})
31
+ expect(ddb).to receive(:put_item).with(table_name: table, item: {'id' => 'fubar', 'perform_at' => 1234}, expected: {id: {exists: false}})
38
32
 
39
- described_class.put_item(table,'perform_at' => 1234)
33
+ described_class.create_item(table,'perform_at' => 1234)
40
34
  end
41
35
  end
42
36
  end
@@ -2,20 +2,17 @@ require 'spec_helper'
2
2
  require 'shoryuken/later/poller'
3
3
 
4
4
  describe Shoryuken::Later::Poller do
5
- let(:ddb_table) { double 'DynamoDb Table' }
6
- let(:ddb_items) { double 'Table Items' }
7
- let(:table) { 'shoryuken_later' }
8
-
5
+ let(:ddb) { double 'DynamoDB' }
9
6
  let(:body) { {'foo' => 'bar'} }
10
7
  let(:json) { JSON.dump(body: body, options: {}) }
8
+ let(:table) { 'shoryuken_later' }
11
9
 
12
10
  let(:ddb_item) do
13
- double AWS::DynamoDB::Item, delete: nil,
14
- attributes: {'id' => 'fubar', 'perform_at' => Time.now + 60, 'shoryuken_args' => json, 'shoryuken_class' => 'TestWorker'}
11
+ {'id' => 'fubar', 'perform_at' => Time.now + 60, 'shoryuken_args' => json, 'shoryuken_class' => 'TestWorker'}
15
12
  end
16
13
 
17
14
  before do
18
- allow(Shoryuken::Later::Client).to receive(:tables).with(table).and_return(ddb_table)
15
+ allow(Shoryuken::Later::Client).to receive(:ddb).and_return(ddb)
19
16
  end
20
17
 
21
18
  subject do
@@ -42,6 +39,8 @@ describe Shoryuken::Later::Poller do
42
39
 
43
40
  describe '#process_item' do
44
41
  it 'enqueues a message if the item could be deleted' do
42
+ allow(Shoryuken::Later::Client).to receive(:delete_item).with(table, ddb_item)
43
+
45
44
  expect(TestWorker).to receive(:perform_in).once do |time,body,options|
46
45
  expect(time ).to be > Time.now
47
46
  expect(body ).to eq(body)
@@ -53,7 +52,7 @@ describe Shoryuken::Later::Poller do
53
52
 
54
53
  it 'does not enqueue a message if the item could not be deleted' do
55
54
  expect(TestWorker).not_to receive(:perform_in)
56
- allow(ddb_item).to receive(:delete) { raise AWS::DynamoDB::Errors::ConditionalCheckFailedException }
55
+ expect(Shoryuken::Later::Client).to receive(:delete_item).with(table, ddb_item){ raise AWS::DynamoDB::Errors::ConditionalCheckFailedException }
57
56
 
58
57
  subject.send(:process_item, ddb_item)
59
58
  end
@@ -21,7 +21,7 @@ describe 'Shoryuken::Worker' do
21
21
  json = JSON.dump(body: 'message', options: {})
22
22
  future = Time.now + 15 * 60 + 1
23
23
 
24
- expect(Shoryuken::Later::Client).to receive(:put_item) do |table,attrs|
24
+ expect(Shoryuken::Later::Client).to receive(:create_item) do |table,attrs|
25
25
  expect(attrs[:perform_at]).to be >= future.to_i
26
26
  expect(attrs[:shoryuken_args]).to eq(json)
27
27
  expect(attrs[:shoryuken_class]).to eq('TestWorker')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: shoryuken-later
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joe Khoobyar
@@ -73,45 +73,59 @@ dependencies:
73
73
  - !ruby/object:Gem::Version
74
74
  version: '0'
75
75
  - !ruby/object:Gem::Dependency
76
- name: aws-sdk-v1
76
+ name: timers
77
77
  requirement: !ruby/object:Gem::Requirement
78
78
  requirements:
79
- - - '>='
79
+ - - ~>
80
80
  - !ruby/object:Gem::Version
81
- version: '0'
81
+ version: 4.0.1
82
82
  type: :runtime
83
83
  prerelease: false
84
84
  version_requirements: !ruby/object:Gem::Requirement
85
85
  requirements:
86
- - - '>='
86
+ - - ~>
87
87
  - !ruby/object:Gem::Version
88
- version: '0'
88
+ version: 4.0.1
89
89
  - !ruby/object:Gem::Dependency
90
- name: timers
90
+ name: aws-sdk-core
91
91
  requirement: !ruby/object:Gem::Requirement
92
92
  requirements:
93
- - - ~>
93
+ - - '='
94
94
  - !ruby/object:Gem::Version
95
- version: 4.0.1
95
+ version: 2.0.21
96
96
  type: :runtime
97
97
  prerelease: false
98
98
  version_requirements: !ruby/object:Gem::Requirement
99
99
  requirements:
100
- - - ~>
100
+ - - '='
101
101
  - !ruby/object:Gem::Version
102
- version: 4.0.1
102
+ version: 2.0.21
103
+ - !ruby/object:Gem::Dependency
104
+ name: aws-sdk-resources
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - '='
108
+ - !ruby/object:Gem::Version
109
+ version: 2.0.21.pre
110
+ type: :runtime
111
+ prerelease: false
112
+ version_requirements: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - '='
115
+ - !ruby/object:Gem::Version
116
+ version: 2.0.21.pre
103
117
  - !ruby/object:Gem::Dependency
104
118
  name: shoryuken
105
119
  requirement: !ruby/object:Gem::Requirement
106
120
  requirements:
107
- - - ~>
121
+ - - '>'
108
122
  - !ruby/object:Gem::Version
109
123
  version: 0.0.5
110
124
  type: :runtime
111
125
  prerelease: false
112
126
  version_requirements: !ruby/object:Gem::Requirement
113
127
  requirements:
114
- - - ~>
128
+ - - '>'
115
129
  - !ruby/object:Gem::Version
116
130
  version: 0.0.5
117
131
  description: "\n This gem provides a scheduling plugin (using Dynamo DB) for Shoryuken,