funktor 0.5.0 → 0.6.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +26 -11
  3. data/funktor-testapp/Gemfile.lock +2 -2
  4. data/funktor-testapp/app/services/job_flood.rb +1 -1
  5. data/funktor-testapp/app/workers/single_thread_audit_worker.rb +3 -0
  6. data/funktor-testapp/funktor_config/environment.yml +2 -2
  7. data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml +3 -1
  8. data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
  9. data/funktor-testapp/funktor_config/function_definitions/job_activator.yml +1 -2
  10. data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml +13 -0
  11. data/funktor-testapp/funktor_config/funktor.yml +25 -25
  12. data/funktor-testapp/funktor_config/iam_permissions/{single_thread_queue.yml → low_concurrency_queue.yml} +1 -1
  13. data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +22 -17
  14. data/funktor-testapp/funktor_config/resources/default_queue.yml +2 -2
  15. data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml +2 -2
  16. data/funktor-testapp/funktor_config/resources/jobs_table.yml +16 -4
  17. data/funktor-testapp/funktor_config/resources/low_concurrency_queue.yml +22 -0
  18. data/funktor-testapp/funktor_init.yml +16 -8
  19. data/funktor-testapp/lambda_event_handlers/{single_thread_queue_handler.rb → low_concurrency_queue_handler.rb} +0 -0
  20. data/funktor-testapp/serverless.yml +4 -3
  21. data/funktor.gemspec +3 -1
  22. data/lib/funktor/activity_tracker.rb +6 -2
  23. data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
  24. data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml +1 -2
  25. data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml +3 -1
  26. data/lib/funktor/cli/templates/funktor_config/funktor.yml +6 -6
  27. data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +3 -2
  28. data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml +2 -2
  29. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +16 -4
  30. data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml +2 -2
  31. data/lib/funktor/cli/templates/funktor_init.yml.tt +14 -8
  32. data/lib/funktor/cli/templates/serverless.yml +1 -0
  33. data/lib/funktor/incoming_job_handler.rb +11 -15
  34. data/lib/funktor/job.rb +50 -5
  35. data/lib/funktor/job_activator.rb +52 -26
  36. data/lib/funktor/shard_utils.rb +6 -0
  37. data/lib/funktor/testing.rb +1 -0
  38. data/lib/funktor/version.rb +1 -1
  39. data/lib/funktor/web/application.rb +139 -0
  40. data/lib/funktor/web/views/index.erb +3 -0
  41. data/lib/funktor/web/views/layout.erb +58 -0
  42. data/lib/funktor/web/views/processing.erb +29 -0
  43. data/lib/funktor/web/views/queued.erb +29 -0
  44. data/lib/funktor/web/views/retries.erb +35 -0
  45. data/lib/funktor/web/views/scheduled.erb +26 -0
  46. data/lib/funktor/web/views/stats.erb +9 -0
  47. data/lib/funktor/web/views/table_stats_with_buttons.erb +11 -0
  48. data/lib/funktor/web.rb +1 -0
  49. data/lib/funktor/work_queue_handler.rb +41 -0
  50. data/lib/funktor/worker/funktor_options.rb +3 -1
  51. data/lib/funktor/worker.rb +8 -11
  52. data/lib/funktor.rb +16 -16
  53. metadata +46 -6
  54. data/funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml +0 -11
  55. data/funktor-testapp/funktor_config/resources/single_thread_queue.yml +0 -22
@@ -0,0 +1,29 @@
1
+ <h1>Processing</h1>
2
+ <p>Showing <%= jobs.count %> of <%= @activity_stats["processing"] %></p>
3
+ <table role="grid">
4
+ <tr>
5
+ <td>ID</td>
6
+ <td>Perform at</td>
7
+ <td>Worker</td>
8
+ <td>Queue</td>
9
+ <td>Params</td>
10
+ <td>Attempts</td>
11
+ <td>Last error</td>
12
+ </tr>
13
+ <% jobs.each do |job| %>
14
+ <tr>
15
+ <td><%= job.job_id %></td>
16
+ <td><%= job.perform_at %></td>
17
+ <td><%= job.worker_class_name %></td>
18
+ <td><%= job.queue %></td>
19
+ <td><%= job.worker_params %></td>
20
+ <td><%= job.retries %></td>
21
+ <td>
22
+ <%= job.error_class %> - <%= job.error_message %>
23
+ <br/>
24
+ <%= job.error_backtrace.first %>
25
+ </td>
26
+ </tr>
27
+ <% end %>
28
+ </table>
29
+
@@ -0,0 +1,29 @@
1
+ <h1>Queued</h1>
2
+ <p>Showing <%= jobs.count %> of <%= @activity_stats["queued"] %></p>
3
+ <table role="grid">
4
+ <tr>
5
+ <td>ID</td>
6
+ <td>Perform at</td>
7
+ <td>Worker</td>
8
+ <td>Queue</td>
9
+ <td>Params</td>
10
+ <td>Attempts</td>
11
+ <td>Last error</td>
12
+ </tr>
13
+ <% jobs.each do |job| %>
14
+ <tr>
15
+ <td><%= job.job_id %></td>
16
+ <td><%= job.perform_at %></td>
17
+ <td><%= job.worker_class_name %></td>
18
+ <td><%= job.queue %></td>
19
+ <td><%= job.worker_params %></td>
20
+ <td><%= job.retries %></td>
21
+ <td>
22
+ <%= job.error_class %> - <%= job.error_message %>
23
+ <br/>
24
+ <%= job.error_backtrace.first %>
25
+ </td>
26
+ </tr>
27
+ <% end %>
28
+ </table>
29
+
@@ -0,0 +1,35 @@
1
+ <h1>Retries</h1>
2
+ <form action="/update_jobs" method="post" accept-charset="utf-8">
3
+ <input type="hidden" name="source" value="retry" />
4
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
5
+ <table role="grid">
6
+ <tr>
7
+ <td>&nbsp;</td>
8
+ <td>ID</td>
9
+ <td>Perform at</td>
10
+ <td>Worker</td>
11
+ <td>Queue</td>
12
+ <td>Params</td>
13
+ <td>Attempts</td>
14
+ <td>Last error</td>
15
+ </tr>
16
+ <% jobs.each do |job| %>
17
+ <tr>
18
+ <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
19
+ <td><%= job.job_id %></td>
20
+ <td><%= job.perform_at %></td>
21
+ <td><%= job.worker_class_name %></td>
22
+ <td><%= job.queue %></td>
23
+ <td><%= job.worker_params %></td>
24
+ <td><%= job.retries %></td>
25
+ <td>
26
+ <%= job.error_class %> - <%= job.error_message %>
27
+ <br/>
28
+ <%= job.error_backtrace.first %>
29
+ </td>
30
+ </tr>
31
+ <% end %>
32
+ </table>
33
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
34
+ </form>
35
+
@@ -0,0 +1,26 @@
1
+ <h1>Scheduled</h1>
2
+ <form action="/update_jobs" method="post" accept-charset="utf-8">
3
+ <input type="hidden" name="source" value="scheduled" />
4
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
5
+ <table role="grid">
6
+ <tr>
7
+ <td>&nbsp;</td>
8
+ <td>ID</td>
9
+ <td>Perform at</td>
10
+ <td>Worker</td>
11
+ <td>Queue</td>
12
+ <td>Params</td>
13
+ </tr>
14
+ <% jobs.each do |job| %>
15
+ <tr>
16
+ <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
17
+ <td><%= job.job_id %></td>
18
+ <td><%= job.perform_at %></td>
19
+ <td><%= job.worker_class_name %></td>
20
+ <td><%= job.queue %></td>
21
+ <td><%= job.worker_params %></td>
22
+ </tr>
23
+ <% end %>
24
+ </table>
25
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
26
+ </form>
@@ -0,0 +1,9 @@
1
+ <table class="header">
2
+ <tr>
3
+ <td><h5>Funktor</h5></td>
4
+ <% @activity_stats.each_pair do |stat, value| %>
5
+ <td><a href="./<%= stat %>"><%= stat %>: <%= value %></a></td>
6
+ <% end %>
7
+ <td>Current time UTC: <%= Time.now.utc %></td>
8
+ </tr>
9
+ </table>
@@ -0,0 +1,11 @@
1
+ <div class="grid">
2
+ <div>
3
+ Showing <%= jobs.count %> of <%= @activity_stats[stat_category] %>
4
+ </div>
5
+ <div>
6
+ <input type="submit" name="submit" value="Delete Selected Jobs" />
7
+ </div>
8
+ <div>
9
+ <input type="submit" name="submit" value="Queue Selected Jobs" />
10
+ </div>
11
+ </div>
@@ -0,0 +1 @@
1
+ require 'funktor/web/application'
@@ -1,4 +1,5 @@
1
1
  require 'aws-sdk-sqs'
2
+ require 'aws-sdk-dynamodb'
2
3
 
3
4
  module Funktor
4
5
  class WorkQueueHandler
@@ -18,6 +19,10 @@ module Funktor
18
19
  end
19
20
  end
20
21
 
22
+ def dynamodb_client
23
+ @dynamodb_client ||= ::Aws::DynamoDB::Client.new
24
+ end
25
+
21
26
  def sqs_client
22
27
  @sqs_client ||= ::Aws::SQS::Client.new
23
28
  end
@@ -25,20 +30,25 @@ module Funktor
25
30
  def dispatch(job)
26
31
  begin
27
32
  @tracker.track(:processingStarted, job)
33
+ update_job_category(job, "processing")
28
34
  Funktor.work_queue_handler_middleware.invoke(job) do
29
35
  job.execute
30
36
  end
31
37
  @processed_counter.incr(job)
32
38
  @tracker.track(:processingComplete, job)
39
+ delete_job_from_dynamodb(job)
33
40
  # rescue Funktor::Job::InvalidJsonError # TODO Make this work
34
41
  rescue Exception => e
35
42
  handle_error(e, job)
36
43
  @failed_counter.incr(job)
44
+ job.error = e
37
45
  if job.can_retry
38
46
  @tracker.track(:retrying, job)
47
+ update_job_category(job, "retry")
39
48
  trigger_retry(job)
40
49
  else
41
50
  @tracker.track(:bailingOut, job)
51
+ update_job_category(job, "dead")
42
52
  Funktor.logger.error "We retried max times. We're bailing on this one."
43
53
  Funktor.logger.error job.to_json
44
54
  end
@@ -56,5 +66,36 @@ module Funktor
56
66
  })
57
67
  end
58
68
 
69
+ def delayed_job_table
70
+ ENV['FUNKTOR_JOBS_TABLE']
71
+ end
72
+
73
+ def update_job_category(job, category)
74
+ dynamodb_client.update_item({
75
+ key: {
76
+ "jobShard" => job.shard,
77
+ "jobId" => job.job_id
78
+ },
79
+ table_name: delayed_job_table,
80
+ update_expression: "SET category = :category, queueable = :queueable",
81
+ expression_attribute_values: {
82
+ ":queueable" => "false",
83
+ ":category" => category
84
+ },
85
+ return_values: "ALL_OLD"
86
+ })
87
+ end
88
+
89
+ def delete_job_from_dynamodb(job)
90
+ dynamodb_client.delete_item({
91
+ key: {
92
+ "jobShard" => job.shard,
93
+ "jobId" => job.job_id
94
+ },
95
+ table_name: delayed_job_table,
96
+ return_values: "ALL_OLD"
97
+ })
98
+ end
99
+
59
100
  end
60
101
  end
@@ -1,10 +1,12 @@
1
+ require 'active_support/core_ext/class/attribute'
2
+
1
3
  module Funktor
2
4
  module Worker
3
5
  module FunktorOptions
4
6
  def self.included(base)
5
7
  base.extend ClassMethods
6
8
  base.class_eval do
7
- cattr_accessor :funktor_options_hash
9
+ class_attribute :funktor_options_hash
8
10
  end
9
11
  end
10
12
  module ClassMethods
@@ -18,30 +18,27 @@ module Funktor::Worker
18
18
  self.perform_in(0, *worker_params)
19
19
  end
20
20
 
21
- def perform_at(time, *worker_params)
22
- delay = (time.utc - Time.now.utc).round
23
- if delay < 0
24
- delay = 0
25
- end
26
- self.perform_in(delay, *worker_params)
21
+ def perform_at(job_time, *worker_params)
22
+ self.push(job_time, *worker_params)
27
23
  end
28
24
 
29
25
  def perform_in(delay, *worker_params)
30
- self.push(delay, *worker_params)
26
+ job_time = Time.now.utc + delay
27
+ self.perform_at(job_time, *worker_params)
31
28
  end
32
29
 
33
- def push(delay, *worker_params)
34
- payload = build_job_payload(delay, *worker_params)
30
+ def push(job_time, *worker_params)
31
+ payload = build_job_payload(job_time, *worker_params)
35
32
  Funktor.job_pusher.push(payload)
36
33
  end
37
34
 
38
- def build_job_payload(delay, *worker_params)
35
+ def build_job_payload(job_time, *worker_params)
39
36
  {
40
37
  worker: self.name,
41
38
  worker_params: worker_params,
42
39
  queue: self.work_queue,
43
40
  incoming_job_queue_url: self.queue_url,
44
- delay: delay,
41
+ perform_at: job_time.utc,
45
42
  funktor_options: get_funktor_options
46
43
  }
47
44
  end
data/lib/funktor.rb CHANGED
@@ -1,15 +1,15 @@
1
- require "funktor/version"
2
- require 'funktor/aws/sqs/event'
3
- require 'funktor/aws/sqs/record'
4
- require 'funktor/counter'
5
- require 'funktor/job'
6
- require 'funktor/job_pusher'
7
- require 'funktor/logger'
8
- require 'funktor/worker'
9
- require 'funktor/middleware_chain'
10
- require 'funktor/incoming_job_handler'
11
- require 'funktor/job_activator'
12
- require 'funktor/activity_tracker'
1
+ require_relative "funktor/version"
2
+ require_relative 'funktor/aws/sqs/event'
3
+ require_relative 'funktor/aws/sqs/record'
4
+ require_relative 'funktor/counter'
5
+ require_relative 'funktor/job'
6
+ require_relative 'funktor/job_pusher'
7
+ require_relative 'funktor/logger'
8
+ require_relative 'funktor/worker'
9
+ require_relative 'funktor/middleware_chain'
10
+ require_relative 'funktor/incoming_job_handler'
11
+ require_relative 'funktor/job_activator'
12
+ require_relative 'funktor/activity_tracker'
13
13
 
14
14
  require 'json'
15
15
 
@@ -132,8 +132,8 @@ end
132
132
  # the main Funktor module is defined?
133
133
  #
134
134
  # TODO - Should we require metrics by default or let people opt in?
135
- require 'funktor/middleware/metrics'
136
- require 'funktor/error_handler'
137
- require 'funktor/work_queue_handler'
135
+ require_relative 'funktor/middleware/metrics'
136
+ require_relative 'funktor/error_handler'
137
+ require_relative 'funktor/work_queue_handler'
138
138
 
139
- require 'funktor/rails' if defined?(::Rails::Engine)
139
+ require_relative 'funktor/rails' if defined?(::Rails::Engine)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-12 00:00:00.000000000 Z
11
+ date: 2021-08-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -122,6 +122,34 @@ dependencies:
122
122
  - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: sinatra
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: timecop
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
125
153
  description: Background processing in AWS Lambda.
126
154
  email:
127
155
  - jeremy@octolabs.com
@@ -154,21 +182,22 @@ files:
154
182
  - funktor-testapp/app/workers/audit_worker.rb
155
183
  - funktor-testapp/app/workers/greetings_worker.rb
156
184
  - funktor-testapp/app/workers/hello_worker.rb
185
+ - funktor-testapp/app/workers/single_thread_audit_worker.rb
157
186
  - funktor-testapp/deploy-dev.sh
158
187
  - funktor-testapp/funktor_config/boot.rb
159
188
  - funktor-testapp/funktor_config/environment.yml
160
189
  - funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml
161
190
  - funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml
162
191
  - funktor-testapp/funktor_config/function_definitions/job_activator.yml
192
+ - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
163
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
164
- - funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml
165
194
  - funktor-testapp/funktor_config/funktor.yml
166
195
  - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
167
196
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
168
197
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
169
198
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
170
199
  - funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
171
- - funktor-testapp/funktor_config/iam_permissions/single_thread_queue.yml
200
+ - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
172
201
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
173
202
  - funktor-testapp/funktor_config/package.yml
174
203
  - funktor-testapp/funktor_config/resources/activity_table.yml
@@ -177,14 +206,14 @@ files:
177
206
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
178
207
  - funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
179
208
  - funktor-testapp/funktor_config/resources/jobs_table.yml
180
- - funktor-testapp/funktor_config/resources/single_thread_queue.yml
209
+ - funktor-testapp/funktor_config/resources/low_concurrency_queue.yml
181
210
  - funktor-testapp/funktor_config/ruby_layer.yml
182
211
  - funktor-testapp/funktor_init.yml
183
212
  - funktor-testapp/lambda_event_handlers/default_queue_handler.rb
184
213
  - funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
185
214
  - funktor-testapp/lambda_event_handlers/job_activator.rb
215
+ - funktor-testapp/lambda_event_handlers/low_concurrency_queue_handler.rb
186
216
  - funktor-testapp/lambda_event_handlers/random_job_generator.rb
187
- - funktor-testapp/lambda_event_handlers/single_thread_queue_handler.rb
188
217
  - funktor-testapp/package-lock.json
189
218
  - funktor-testapp/package.json
190
219
  - funktor-testapp/serverless.yml
@@ -240,8 +269,19 @@ files:
240
269
  - lib/funktor/middleware/metrics.rb
241
270
  - lib/funktor/middleware_chain.rb
242
271
  - lib/funktor/rails.rb
272
+ - lib/funktor/shard_utils.rb
243
273
  - lib/funktor/testing.rb
244
274
  - lib/funktor/version.rb
275
+ - lib/funktor/web.rb
276
+ - lib/funktor/web/application.rb
277
+ - lib/funktor/web/views/index.erb
278
+ - lib/funktor/web/views/layout.erb
279
+ - lib/funktor/web/views/processing.erb
280
+ - lib/funktor/web/views/queued.erb
281
+ - lib/funktor/web/views/retries.erb
282
+ - lib/funktor/web/views/scheduled.erb
283
+ - lib/funktor/web/views/stats.erb
284
+ - lib/funktor/web/views/table_stats_with_buttons.erb
245
285
  - lib/funktor/work_queue_handler.rb
246
286
  - lib/funktor/worker.rb
247
287
  - lib/funktor/worker/funktor_options.rb
@@ -1,11 +0,0 @@
1
- handler: lambda_event_handlers/single_thread_queue_handler.call
2
- timeout: ${self:custom.funktor.SingleThreadQueueHandler.timeout, 900}
3
- reservedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.reservedConcurrency, null}
4
- provisionedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.provisionedConcurrency, null}
5
- memorySize: ${self:custom.funktor.SingleThreadQueueHandler.memorySize, 256}
6
- events:
7
- - sqs:
8
- arn:
9
- Fn::GetAtt:
10
- - SingleThreadQueue
11
- - Arn