funktor 0.5.0 → 0.6.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +26 -11
  3. data/funktor-testapp/Gemfile.lock +2 -2
  4. data/funktor-testapp/app/services/job_flood.rb +1 -1
  5. data/funktor-testapp/app/workers/single_thread_audit_worker.rb +3 -0
  6. data/funktor-testapp/funktor_config/environment.yml +2 -2
  7. data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml +3 -1
  8. data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
  9. data/funktor-testapp/funktor_config/function_definitions/job_activator.yml +1 -2
  10. data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml +13 -0
  11. data/funktor-testapp/funktor_config/funktor.yml +25 -25
  12. data/funktor-testapp/funktor_config/iam_permissions/{single_thread_queue.yml → low_concurrency_queue.yml} +1 -1
  13. data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +22 -17
  14. data/funktor-testapp/funktor_config/resources/default_queue.yml +2 -2
  15. data/funktor-testapp/funktor_config/resources/incoming_job_queue.yml +2 -2
  16. data/funktor-testapp/funktor_config/resources/jobs_table.yml +16 -4
  17. data/funktor-testapp/funktor_config/resources/low_concurrency_queue.yml +22 -0
  18. data/funktor-testapp/funktor_init.yml +16 -8
  19. data/funktor-testapp/lambda_event_handlers/{single_thread_queue_handler.rb → low_concurrency_queue_handler.rb} +0 -0
  20. data/funktor-testapp/serverless.yml +4 -3
  21. data/funktor.gemspec +3 -1
  22. data/lib/funktor/activity_tracker.rb +6 -2
  23. data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml +3 -1
  24. data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml +1 -2
  25. data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml +3 -1
  26. data/lib/funktor/cli/templates/funktor_config/funktor.yml +6 -6
  27. data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +3 -2
  28. data/lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml +2 -2
  29. data/lib/funktor/cli/templates/funktor_config/resources/jobs_table.yml +16 -4
  30. data/lib/funktor/cli/templates/funktor_config/resources/work_queue.yml +2 -2
  31. data/lib/funktor/cli/templates/funktor_init.yml.tt +14 -8
  32. data/lib/funktor/cli/templates/serverless.yml +1 -0
  33. data/lib/funktor/incoming_job_handler.rb +11 -15
  34. data/lib/funktor/job.rb +50 -5
  35. data/lib/funktor/job_activator.rb +52 -26
  36. data/lib/funktor/shard_utils.rb +6 -0
  37. data/lib/funktor/testing.rb +1 -0
  38. data/lib/funktor/version.rb +1 -1
  39. data/lib/funktor/web/application.rb +139 -0
  40. data/lib/funktor/web/views/index.erb +3 -0
  41. data/lib/funktor/web/views/layout.erb +58 -0
  42. data/lib/funktor/web/views/processing.erb +29 -0
  43. data/lib/funktor/web/views/queued.erb +29 -0
  44. data/lib/funktor/web/views/retries.erb +35 -0
  45. data/lib/funktor/web/views/scheduled.erb +26 -0
  46. data/lib/funktor/web/views/stats.erb +9 -0
  47. data/lib/funktor/web/views/table_stats_with_buttons.erb +11 -0
  48. data/lib/funktor/web.rb +1 -0
  49. data/lib/funktor/work_queue_handler.rb +41 -0
  50. data/lib/funktor/worker/funktor_options.rb +3 -1
  51. data/lib/funktor/worker.rb +8 -11
  52. data/lib/funktor.rb +16 -16
  53. metadata +46 -6
  54. data/funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml +0 -11
  55. data/funktor-testapp/funktor_config/resources/single_thread_queue.yml +0 -22
@@ -0,0 +1,29 @@
1
+ <h1>Processing</h1>
2
+ <p>Showing <%= jobs.count %> of <%= @activity_stats["processing"] %></p>
3
+ <table role="grid">
4
+ <tr>
5
+ <td>ID</td>
6
+ <td>Perform at</td>
7
+ <td>Worker</td>
8
+ <td>Queue</td>
9
+ <td>Params</td>
10
+ <td>Attempts</td>
11
+ <td>Last error</td>
12
+ </tr>
13
+ <% jobs.each do |job| %>
14
+ <tr>
15
+ <td><%= job.job_id %></td>
16
+ <td><%= job.perform_at %></td>
17
+ <td><%= job.worker_class_name %></td>
18
+ <td><%= job.queue %></td>
19
+ <td><%= job.worker_params %></td>
20
+ <td><%= job.retries %></td>
21
+ <td>
22
+ <%= job.error_class %> - <%= job.error_message %>
23
+ <br/>
24
+ <%= job.error_backtrace.first %>
25
+ </td>
26
+ </tr>
27
+ <% end %>
28
+ </table>
29
+
@@ -0,0 +1,29 @@
1
+ <h1>Queued</h1>
2
+ <p>Showing <%= jobs.count %> of <%= @activity_stats["queued"] %></p>
3
+ <table role="grid">
4
+ <tr>
5
+ <td>ID</td>
6
+ <td>Perform at</td>
7
+ <td>Worker</td>
8
+ <td>Queue</td>
9
+ <td>Params</td>
10
+ <td>Attempts</td>
11
+ <td>Last error</td>
12
+ </tr>
13
+ <% jobs.each do |job| %>
14
+ <tr>
15
+ <td><%= job.job_id %></td>
16
+ <td><%= job.perform_at %></td>
17
+ <td><%= job.worker_class_name %></td>
18
+ <td><%= job.queue %></td>
19
+ <td><%= job.worker_params %></td>
20
+ <td><%= job.retries %></td>
21
+ <td>
22
+ <%= job.error_class %> - <%= job.error_message %>
23
+ <br/>
24
+ <%= job.error_backtrace.first %>
25
+ </td>
26
+ </tr>
27
+ <% end %>
28
+ </table>
29
+
@@ -0,0 +1,35 @@
1
+ <h1>Retries</h1>
2
+ <form action="/update_jobs" method="post" accept-charset="utf-8">
3
+ <input type="hidden" name="source" value="retry" />
4
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
5
+ <table role="grid">
6
+ <tr>
7
+ <td>&nbsp;</td>
8
+ <td>ID</td>
9
+ <td>Perform at</td>
10
+ <td>Worker</td>
11
+ <td>Queue</td>
12
+ <td>Params</td>
13
+ <td>Attempts</td>
14
+ <td>Last error</td>
15
+ </tr>
16
+ <% jobs.each do |job| %>
17
+ <tr>
18
+ <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
19
+ <td><%= job.job_id %></td>
20
+ <td><%= job.perform_at %></td>
21
+ <td><%= job.worker_class_name %></td>
22
+ <td><%= job.queue %></td>
23
+ <td><%= job.worker_params %></td>
24
+ <td><%= job.retries %></td>
25
+ <td>
26
+ <%= job.error_class %> - <%= job.error_message %>
27
+ <br/>
28
+ <%= job.error_backtrace.first %>
29
+ </td>
30
+ </tr>
31
+ <% end %>
32
+ </table>
33
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'retries' } %>
34
+ </form>
35
+
@@ -0,0 +1,26 @@
1
+ <h1>Scheduled</h1>
2
+ <form action="/update_jobs" method="post" accept-charset="utf-8">
3
+ <input type="hidden" name="source" value="scheduled" />
4
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
5
+ <table role="grid">
6
+ <tr>
7
+ <td>&nbsp;</td>
8
+ <td>ID</td>
9
+ <td>Perform at</td>
10
+ <td>Worker</td>
11
+ <td>Queue</td>
12
+ <td>Params</td>
13
+ </tr>
14
+ <% jobs.each do |job| %>
15
+ <tr>
16
+ <td><input type="checkbox" name="job_id[]" value="<%= job.job_id %>" id=""></td>
17
+ <td><%= job.job_id %></td>
18
+ <td><%= job.perform_at %></td>
19
+ <td><%= job.worker_class_name %></td>
20
+ <td><%= job.queue %></td>
21
+ <td><%= job.worker_params %></td>
22
+ </tr>
23
+ <% end %>
24
+ </table>
25
+ <%= erb :table_stats_with_buttons, locals: { jobs: jobs, stat_category: 'scheduled' } %>
26
+ </form>
@@ -0,0 +1,9 @@
1
+ <table class="header">
2
+ <tr>
3
+ <td><h5>Funktor</h5></td>
4
+ <% @activity_stats.each_pair do |stat, value| %>
5
+ <td><a href="./<%= stat %>"><%= stat %>: <%= value %></a></td>
6
+ <% end %>
7
+ <td>Current time UTC: <%= Time.now.utc %></td>
8
+ </tr>
9
+ </table>
@@ -0,0 +1,11 @@
1
+ <div class="grid">
2
+ <div>
3
+ Showing <%= jobs.count %> of <%= @activity_stats[stat_category] %>
4
+ </div>
5
+ <div>
6
+ <input type="submit" name="submit" value="Delete Selected Jobs" />
7
+ </div>
8
+ <div>
9
+ <input type="submit" name="submit" value="Queue Selected Jobs" />
10
+ </div>
11
+ </div>
@@ -0,0 +1 @@
1
+ require 'funktor/web/application'
@@ -1,4 +1,5 @@
1
1
  require 'aws-sdk-sqs'
2
+ require 'aws-sdk-dynamodb'
2
3
 
3
4
  module Funktor
4
5
  class WorkQueueHandler
@@ -18,6 +19,10 @@ module Funktor
18
19
  end
19
20
  end
20
21
 
22
+ def dynamodb_client
23
+ @dynamodb_client ||= ::Aws::DynamoDB::Client.new
24
+ end
25
+
21
26
  def sqs_client
22
27
  @sqs_client ||= ::Aws::SQS::Client.new
23
28
  end
@@ -25,20 +30,25 @@ module Funktor
25
30
  def dispatch(job)
26
31
  begin
27
32
  @tracker.track(:processingStarted, job)
33
+ update_job_category(job, "processing")
28
34
  Funktor.work_queue_handler_middleware.invoke(job) do
29
35
  job.execute
30
36
  end
31
37
  @processed_counter.incr(job)
32
38
  @tracker.track(:processingComplete, job)
39
+ delete_job_from_dynamodb(job)
33
40
  # rescue Funktor::Job::InvalidJsonError # TODO Make this work
34
41
  rescue Exception => e
35
42
  handle_error(e, job)
36
43
  @failed_counter.incr(job)
44
+ job.error = e
37
45
  if job.can_retry
38
46
  @tracker.track(:retrying, job)
47
+ update_job_category(job, "retry")
39
48
  trigger_retry(job)
40
49
  else
41
50
  @tracker.track(:bailingOut, job)
51
+ update_job_category(job, "dead")
42
52
  Funktor.logger.error "We retried max times. We're bailing on this one."
43
53
  Funktor.logger.error job.to_json
44
54
  end
@@ -56,5 +66,36 @@ module Funktor
56
66
  })
57
67
  end
58
68
 
69
+ def delayed_job_table
70
+ ENV['FUNKTOR_JOBS_TABLE']
71
+ end
72
+
73
+ def update_job_category(job, category)
74
+ dynamodb_client.update_item({
75
+ key: {
76
+ "jobShard" => job.shard,
77
+ "jobId" => job.job_id
78
+ },
79
+ table_name: delayed_job_table,
80
+ update_expression: "SET category = :category, queueable = :queueable",
81
+ expression_attribute_values: {
82
+ ":queueable" => "false",
83
+ ":category" => category
84
+ },
85
+ return_values: "ALL_OLD"
86
+ })
87
+ end
88
+
89
+ def delete_job_from_dynamodb(job)
90
+ dynamodb_client.delete_item({
91
+ key: {
92
+ "jobShard" => job.shard,
93
+ "jobId" => job.job_id
94
+ },
95
+ table_name: delayed_job_table,
96
+ return_values: "ALL_OLD"
97
+ })
98
+ end
99
+
59
100
  end
60
101
  end
@@ -1,10 +1,12 @@
1
+ require 'active_support/core_ext/class/attribute'
2
+
1
3
  module Funktor
2
4
  module Worker
3
5
  module FunktorOptions
4
6
  def self.included(base)
5
7
  base.extend ClassMethods
6
8
  base.class_eval do
7
- cattr_accessor :funktor_options_hash
9
+ class_attribute :funktor_options_hash
8
10
  end
9
11
  end
10
12
  module ClassMethods
@@ -18,30 +18,27 @@ module Funktor::Worker
18
18
  self.perform_in(0, *worker_params)
19
19
  end
20
20
 
21
- def perform_at(time, *worker_params)
22
- delay = (time.utc - Time.now.utc).round
23
- if delay < 0
24
- delay = 0
25
- end
26
- self.perform_in(delay, *worker_params)
21
+ def perform_at(job_time, *worker_params)
22
+ self.push(job_time, *worker_params)
27
23
  end
28
24
 
29
25
  def perform_in(delay, *worker_params)
30
- self.push(delay, *worker_params)
26
+ job_time = Time.now.utc + delay
27
+ self.perform_at(job_time, *worker_params)
31
28
  end
32
29
 
33
- def push(delay, *worker_params)
34
- payload = build_job_payload(delay, *worker_params)
30
+ def push(job_time, *worker_params)
31
+ payload = build_job_payload(job_time, *worker_params)
35
32
  Funktor.job_pusher.push(payload)
36
33
  end
37
34
 
38
- def build_job_payload(delay, *worker_params)
35
+ def build_job_payload(job_time, *worker_params)
39
36
  {
40
37
  worker: self.name,
41
38
  worker_params: worker_params,
42
39
  queue: self.work_queue,
43
40
  incoming_job_queue_url: self.queue_url,
44
- delay: delay,
41
+ perform_at: job_time.utc,
45
42
  funktor_options: get_funktor_options
46
43
  }
47
44
  end
data/lib/funktor.rb CHANGED
@@ -1,15 +1,15 @@
1
- require "funktor/version"
2
- require 'funktor/aws/sqs/event'
3
- require 'funktor/aws/sqs/record'
4
- require 'funktor/counter'
5
- require 'funktor/job'
6
- require 'funktor/job_pusher'
7
- require 'funktor/logger'
8
- require 'funktor/worker'
9
- require 'funktor/middleware_chain'
10
- require 'funktor/incoming_job_handler'
11
- require 'funktor/job_activator'
12
- require 'funktor/activity_tracker'
1
+ require_relative "funktor/version"
2
+ require_relative 'funktor/aws/sqs/event'
3
+ require_relative 'funktor/aws/sqs/record'
4
+ require_relative 'funktor/counter'
5
+ require_relative 'funktor/job'
6
+ require_relative 'funktor/job_pusher'
7
+ require_relative 'funktor/logger'
8
+ require_relative 'funktor/worker'
9
+ require_relative 'funktor/middleware_chain'
10
+ require_relative 'funktor/incoming_job_handler'
11
+ require_relative 'funktor/job_activator'
12
+ require_relative 'funktor/activity_tracker'
13
13
 
14
14
  require 'json'
15
15
 
@@ -132,8 +132,8 @@ end
132
132
  # the main Funktor module is defined?
133
133
  #
134
134
  # TODO - Should we require metrics by default or let people opt in?
135
- require 'funktor/middleware/metrics'
136
- require 'funktor/error_handler'
137
- require 'funktor/work_queue_handler'
135
+ require_relative 'funktor/middleware/metrics'
136
+ require_relative 'funktor/error_handler'
137
+ require_relative 'funktor/work_queue_handler'
138
138
 
139
- require 'funktor/rails' if defined?(::Rails::Engine)
139
+ require_relative 'funktor/rails' if defined?(::Rails::Engine)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: funktor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.0
4
+ version: 0.6.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Green
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-08-12 00:00:00.000000000 Z
11
+ date: 2021-08-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk-sqs
@@ -122,6 +122,34 @@ dependencies:
122
122
  - - ">="
123
123
  - !ruby/object:Gem::Version
124
124
  version: '0'
125
+ - !ruby/object:Gem::Dependency
126
+ name: sinatra
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - ">="
130
+ - !ruby/object:Gem::Version
131
+ version: '0'
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - ">="
137
+ - !ruby/object:Gem::Version
138
+ version: '0'
139
+ - !ruby/object:Gem::Dependency
140
+ name: timecop
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - ">="
144
+ - !ruby/object:Gem::Version
145
+ version: '0'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ version: '0'
125
153
  description: Background processing in AWS Lambda.
126
154
  email:
127
155
  - jeremy@octolabs.com
@@ -154,21 +182,22 @@ files:
154
182
  - funktor-testapp/app/workers/audit_worker.rb
155
183
  - funktor-testapp/app/workers/greetings_worker.rb
156
184
  - funktor-testapp/app/workers/hello_worker.rb
185
+ - funktor-testapp/app/workers/single_thread_audit_worker.rb
157
186
  - funktor-testapp/deploy-dev.sh
158
187
  - funktor-testapp/funktor_config/boot.rb
159
188
  - funktor-testapp/funktor_config/environment.yml
160
189
  - funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml
161
190
  - funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml
162
191
  - funktor-testapp/funktor_config/function_definitions/job_activator.yml
192
+ - funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml
163
193
  - funktor-testapp/funktor_config/function_definitions/random_job_generator.yml
164
- - funktor-testapp/funktor_config/function_definitions/single_thread_queue_handler.yml
165
194
  - funktor-testapp/funktor_config/funktor.yml
166
195
  - funktor-testapp/funktor_config/iam_permissions/activity_table.yml
167
196
  - funktor-testapp/funktor_config/iam_permissions/default_queue.yml
168
197
  - funktor-testapp/funktor_config/iam_permissions/incoming_job_queue.yml
169
198
  - funktor-testapp/funktor_config/iam_permissions/jobs_table.yml
170
199
  - funktor-testapp/funktor_config/iam_permissions/jobs_table_secondary_index.yml
171
- - funktor-testapp/funktor_config/iam_permissions/single_thread_queue.yml
200
+ - funktor-testapp/funktor_config/iam_permissions/low_concurrency_queue.yml
172
201
  - funktor-testapp/funktor_config/iam_permissions/ssm.yml
173
202
  - funktor-testapp/funktor_config/package.yml
174
203
  - funktor-testapp/funktor_config/resources/activity_table.yml
@@ -177,14 +206,14 @@ files:
177
206
  - funktor-testapp/funktor_config/resources/incoming_job_queue.yml
178
207
  - funktor-testapp/funktor_config/resources/incoming_job_queue_user.yml
179
208
  - funktor-testapp/funktor_config/resources/jobs_table.yml
180
- - funktor-testapp/funktor_config/resources/single_thread_queue.yml
209
+ - funktor-testapp/funktor_config/resources/low_concurrency_queue.yml
181
210
  - funktor-testapp/funktor_config/ruby_layer.yml
182
211
  - funktor-testapp/funktor_init.yml
183
212
  - funktor-testapp/lambda_event_handlers/default_queue_handler.rb
184
213
  - funktor-testapp/lambda_event_handlers/incoming_job_handler.rb
185
214
  - funktor-testapp/lambda_event_handlers/job_activator.rb
215
+ - funktor-testapp/lambda_event_handlers/low_concurrency_queue_handler.rb
186
216
  - funktor-testapp/lambda_event_handlers/random_job_generator.rb
187
- - funktor-testapp/lambda_event_handlers/single_thread_queue_handler.rb
188
217
  - funktor-testapp/package-lock.json
189
218
  - funktor-testapp/package.json
190
219
  - funktor-testapp/serverless.yml
@@ -240,8 +269,19 @@ files:
240
269
  - lib/funktor/middleware/metrics.rb
241
270
  - lib/funktor/middleware_chain.rb
242
271
  - lib/funktor/rails.rb
272
+ - lib/funktor/shard_utils.rb
243
273
  - lib/funktor/testing.rb
244
274
  - lib/funktor/version.rb
275
+ - lib/funktor/web.rb
276
+ - lib/funktor/web/application.rb
277
+ - lib/funktor/web/views/index.erb
278
+ - lib/funktor/web/views/layout.erb
279
+ - lib/funktor/web/views/processing.erb
280
+ - lib/funktor/web/views/queued.erb
281
+ - lib/funktor/web/views/retries.erb
282
+ - lib/funktor/web/views/scheduled.erb
283
+ - lib/funktor/web/views/stats.erb
284
+ - lib/funktor/web/views/table_stats_with_buttons.erb
245
285
  - lib/funktor/work_queue_handler.rb
246
286
  - lib/funktor/worker.rb
247
287
  - lib/funktor/worker/funktor_options.rb
@@ -1,11 +0,0 @@
1
- handler: lambda_event_handlers/single_thread_queue_handler.call
2
- timeout: ${self:custom.funktor.SingleThreadQueueHandler.timeout, 900}
3
- reservedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.reservedConcurrency, null}
4
- provisionedConcurrency: ${self:custom.funktor.SingleThreadQueueHandler.provisionedConcurrency, null}
5
- memorySize: ${self:custom.funktor.SingleThreadQueueHandler.memorySize, 256}
6
- events:
7
- - sqs:
8
- arn:
9
- Fn::GetAtt:
10
- - SingleThreadQueue
11
- - Arn