funktor 0.7.5 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +3 -3
  3. data/funktor-testapp/Dockerfile +63 -0
  4. data/funktor-testapp/app/services/job_flood.rb +1 -1
  5. data/funktor-testapp/funktor_config/boot.rb +3 -0
  6. data/funktor-testapp/funktor_config/environment.yml +10 -1
  7. data/funktor-testapp/funktor_config/function_definitions/default_queue_handler.yml +5 -1
  8. data/funktor-testapp/funktor_config/function_definitions/incoming_job_handler.yml +5 -1
  9. data/funktor-testapp/funktor_config/function_definitions/job_activator.yml +5 -1
  10. data/funktor-testapp/funktor_config/function_definitions/low_concurrency_queue_handler.yml +5 -1
  11. data/funktor-testapp/funktor_config/package.yml +6 -6
  12. data/funktor-testapp/funktor_config/resources/cloudwatch_dashboard.yml +645 -37
  13. data/funktor-testapp/funktor_config/ruby_layer.yml +1 -1
  14. data/funktor-testapp/serverless.yml +13 -3
  15. data/lib/funktor/activity_tracker.rb +1 -2
  16. data/lib/funktor/cli/init.rb +4 -0
  17. data/lib/funktor/cli/templates/Dockerfile +63 -0
  18. data/lib/funktor/cli/templates/funktor_config/environment.yml +10 -1
  19. data/lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml +5 -1
  20. data/lib/funktor/cli/templates/funktor_config/function_definitions/job_activator.yml +5 -1
  21. data/lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml +5 -1
  22. data/lib/funktor/cli/templates/funktor_config/package.yml +6 -6
  23. data/lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml +161 -0
  24. data/lib/funktor/cli/templates/funktor_config/ruby_layer.yml +1 -1
  25. data/lib/funktor/cli/templates/serverless.yml +13 -3
  26. data/lib/funktor/incoming_job_handler.rb +9 -7
  27. data/lib/funktor/job_activator.rb +33 -17
  28. data/lib/funktor/job_pusher.rb +1 -1
  29. data/lib/funktor/version.rb +1 -1
  30. data/lib/funktor/web/application.rb +1 -1
  31. data/lib/funktor/work_queue_handler.rb +21 -6
  32. data/lib/funktor.rb +20 -1
  33. metadata +4 -2
@@ -1,4 +1,4 @@
1
- use_docker: false
1
+ #use_docker: false
2
2
  #docker_yums:
3
3
  #- postgresql-devel
4
4
  #native_libs:
@@ -25,6 +25,16 @@ provider:
25
25
  runtime: ruby2.7
26
26
  stage: ${opt:stage, 'dev'}
27
27
  lambdaHashingVersion: 20201221
28
+ ecr:
29
+ scanOnPush: true
30
+ images:
31
+ funktorimage:
32
+ path: ./
33
+ file: Dockerfile
34
+ #platform: linux/amd64 # May be needed if you're on an M1.
35
+ buildArgs:
36
+ # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
37
+ #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
28
38
  environment: ${file(funktor_config/environment.yml)}
29
39
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
30
40
  logRetentionInDays: 7
@@ -42,9 +52,9 @@ custom:
42
52
  # commands. Or fallsback to what we have set in the provider section.
43
53
  stage: ${self:provider.stage, 'dev'}
44
54
  funktor: ${file(funktor_config/funktor.yml)}
45
- rubyLayer: ${file(funktor_config/ruby_layer.yml)}
55
+ #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
46
56
 
47
- package: ${file(funktor_config/package.yml)}
57
+ #package: ${file(funktor_config/package.yml)}
48
58
 
49
59
  functions:
50
60
  IncomingJobHandler: ${file(funktor_config/function_definitions/incoming_job_handler.yml)}
@@ -62,4 +72,4 @@ resources:
62
72
  - ${file(funktor_config/resources/jobs_table.yml)}
63
73
 
64
74
  plugins:
65
- - serverless-ruby-layer
75
+ #- serverless-ruby-layer
@@ -1,5 +1,4 @@
1
1
  require 'json'
2
- require 'aws-sdk-dynamodb'
3
2
 
4
3
  module Funktor
5
4
  class ActivityTracker
@@ -74,7 +73,7 @@ module Funktor
74
73
  end
75
74
 
76
75
  def dynamodb_client
77
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
76
+ Funktor.dynamodb_client
78
77
  end
79
78
 
80
79
  def metric_namespace
@@ -135,6 +135,10 @@ module Funktor
135
135
  template "serverless.yml", File.join("serverless.yml")
136
136
  end
137
137
 
138
+ def dockerfile
139
+ template "Dockerfile", File.join("Dockerfile")
140
+ end
141
+
138
142
  private
139
143
 
140
144
  def app_worker_names
@@ -0,0 +1,63 @@
1
+ FROM public.ecr.aws/lambda/ruby:2.7 AS build_image
2
+
3
+ # Uncomment this as a cache buster
4
+ #RUN echo date
5
+
6
+ # If you supplied buildArgs to the ecr image you can access them here using ARG & ENV
7
+ #ARG BUNDLE_GEM__FURY__IO
8
+ #ENV BUNDLE_GEM__FURY__IO=${BUNDLE_GEM__FURY__IO}
9
+
10
+ # First we install some stuff that we need for gems that have to compile native extensions
11
+ #RUN yum groupinstall "Development Tools" -y
12
+ #RUN yum install -y amazon-linux-extras
13
+ #RUN amazon-linux-extras enable postgresql11
14
+ #RUN yum install -y postgresql-devel
15
+
16
+ # Now we copy the Gemfile and Gemfile.lock into the build image so we can install our gems
17
+ COPY Gemfile Gemfile.lock .
18
+
19
+ # Set a few bundle configuration options
20
+ RUN bundle lock --add-platform x86_64-linux
21
+ #RUN bundle config --local deployment true
22
+ #RUN bundle config --local plugins false
23
+ #RUN bundle config --local frozen true
24
+ #RUN bundle config --local without 'development test'
25
+ RUN bundle config --local path './vendor/bundle'
26
+
27
+ # Now install our gems
28
+ RUN bundle install --quiet --jobs 4
29
+
30
+ # Now we start a second stage in the build that is a clean image without build tools
31
+ FROM public.ecr.aws/lambda/ruby:2.7 AS deploy_image
32
+
33
+ #ENV RAILS_ENV=production
34
+ #ENV RACK_ENV=production
35
+
36
+ # Install node so that asset related gems have a JS runtime.
37
+ # We ship the node executeable to production to make it easier to get an app deployed.
38
+ # TODO: Document steps that could be taken to remove this dependency.
39
+ #RUN curl --silent --location https://rpm.nodesource.com/setup_14.x | bash -
40
+ #RUN yum install -y nodejs
41
+
42
+ # Then copy some postgres related files
43
+ #COPY --from=build_image /usr/lib64/libpq.so.5 /usr/lib64/
44
+ #COPY --from=build_image /usr/lib64/libldap_r-2.4.so.2 /usr/lib64/
45
+ #COPY --from=build_image /usr/lib64/liblber-2.4.so.2 /usr/lib64/
46
+ #COPY --from=build_image /usr/lib64/libsasl2.so.3 /usr/lib64/
47
+ #COPY --from=build_image /usr/lib64/libssl3.so /usr/lib64/
48
+ #COPY --from=build_image /usr/lib64/libsmime3.so /usr/lib64/
49
+ #COPY --from=build_image /usr/lib64/libnss3.so /usr/lib64/
50
+
51
+
52
+ # Next we copy the app from our local directory and we copy
53
+ # the bundled gems from the build image.
54
+ # We do this after copying dependencies becuase the app will
55
+ # change more frequently, and we can used caching up to here.
56
+
57
+ COPY --from=build_image /var/task .
58
+ COPY . .
59
+
60
+ # And finally we have the CMD for the deployed container
61
+ # You can overwrite command in `serverless.yml` template
62
+ CMD [ "app.LambdaFunction::Handler.process" ]
63
+
@@ -1,7 +1,16 @@
1
- BUNDLE_WITHOUT: development:test:assets
1
+ BUNDLE_WITHOUT: development:test
2
2
  BUNDLE_PLUGINS: false
3
3
  BUNDLE_FROZEN: true
4
+ BUNDLE_DEPLOYMENT: true
4
5
  SERVERLESS_STAGE: ${self:custom.stage}
6
+
7
+ FUNKTOR_LOG_LEVEL: INFO
8
+
9
+ RAILS_LOG_TO_STDOUT: true
10
+ RAILS_ENV: production
11
+ RACK_ENV: production
12
+ RAILS_MAX_THREADS: 1
13
+
5
14
  FUNKTOR_APP_NAME: <%= app_name %>
6
15
  FUNKTOR_INCOMING_JOB_QUEUE:
7
16
  Ref: IncomingJobQueue
@@ -1,8 +1,12 @@
1
- handler: lambda_event_handlers/incoming_job_handler.call
1
+ #handler: lambda_event_handlers/incoming_job_handler.call
2
2
  timeout: ${self:custom.funktor.IncomingJobHandler.functionTimeout, 30}
3
3
  reservedConcurrency: ${self:custom.funktor.IncomingJobHandler.reservedConcurrency, null}
4
4
  provisionedConcurrency: ${self:custom.funktor.IncomingJobHandler.provisionedConcurrency, null}
5
5
  memorySize: ${self:custom.funktor.IncomingJobHandler.memorySize, 256}
6
+ image:
7
+ name: funktorimage
8
+ command:
9
+ - lambda_event_handlers/incoming_job_handler.call
6
10
  events:
7
11
  - sqs:
8
12
  batchSize: ${self:custom.funktor.IncomingJobHandler.batchSize, 1}
@@ -1,7 +1,11 @@
1
- handler: lambda_event_handlers/job_activator.call
1
+ #handler: lambda_event_handlers/job_activator.call
2
2
  timeout: ${self:custom.funktor.JobActivator.functionTimeout, 30}
3
3
  reservedConcurrency: ${self:custom.funktor.JobActivator.reservedConcurrency, null}
4
4
  provisionedConcurrency: ${self:custom.funktor.JobActivator.provisionedConcurrency, null}
5
5
  memorySize: ${self:custom.funktor.JobActivator.memorySize, 256}
6
+ image:
7
+ name: funktorimage
8
+ command:
9
+ - lambda_event_handlers/job_activator.call
6
10
  events:
7
11
  - schedule: rate(1 minute)
@@ -1,8 +1,12 @@
1
- handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
1
+ #handler: lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
2
2
  timeout: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.functionTimeout, 900}
3
3
  reservedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.reservedConcurrency, null}
4
4
  provisionedConcurrency: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.provisionedConcurrency, null}
5
5
  memorySize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.memorySize, 256}
6
+ image:
7
+ name: funktorimage
8
+ command:
9
+ - lambda_event_handlers/<%= work_queue_name.underscore %>_queue_handler.call
6
10
  events:
7
11
  - sqs:
8
12
  batchSize: ${self:custom.funktor.<%= work_queue_name.camelize %>QueueHandler.batchSize, 1}
@@ -1,10 +1,10 @@
1
1
  # TODO - Figure out how to allow individual packaging to work out of the box.
2
- individually: false
3
- include:
4
- - Gemfile
5
- - Gemfile.lock
6
- - funktor_config/boot.rb
7
- - app/**
2
+ #individually: false
3
+ #include:
4
+ #- Gemfile
5
+ #- Gemfile.lock
6
+ #- funktor_config/boot.rb
7
+ #- app/**
8
8
  # Evertyting is excluded by default with serverless-ruby-layer, but you could use
9
9
  # the lines below to exlude files that are inside an include path.
10
10
  #exclude:
@@ -21,6 +21,160 @@ Resources:
21
21
  },
22
22
  <% current_y += 3 %>
23
23
 
24
+
25
+ <%- app_worker_names.each do |worker_name| -%>
26
+ { <% "Worker" %>
27
+ "height": 3,
28
+ "width": 6,
29
+ "y": <%= current_y %>,
30
+ "x": 0,
31
+ "type": "text",
32
+ "properties": {
33
+ "markdown": "\n# <%= worker_name %>\n"
34
+ }
35
+ },
36
+ { <% "Worker Jobs per minute" %>
37
+ "height": 3,
38
+ "width": 6,
39
+ "y": <%= current_y %>,
40
+ "x": 6,
41
+ "type": "metric",
42
+ "properties": {
43
+ "metrics": [
44
+ [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ],
45
+ [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" } ]
46
+ ],
47
+ "view": "singleValue",
48
+ "region": "us-east-1",
49
+ "stat": "Sum",
50
+ "period": 60,
51
+ "sparkline": true,
52
+ "title": "<%= worker_name %> Jobs Per Minute"
53
+ }
54
+ },
55
+ { <% "Worker Durations" %>
56
+ "height": 3,
57
+ "width": 12,
58
+ "y": <%= current_y %>,
59
+ "x": 12,
60
+ "type": "metric",
61
+ "properties": {
62
+ "metrics": [
63
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>", { "label": "p10" } ],
64
+ [ "...", { "label": "p50", "stat": "p50" } ],
65
+ [ "...", { "label": "p99", "stat": "p99" } ],
66
+ [ "...", { "label": "Average", "stat": "Average" } ]
67
+ ],
68
+ "view": "singleValue",
69
+ "region": "us-east-1",
70
+ "stat": "p10",
71
+ "period": 60,
72
+ "sparkline": true,
73
+ "title": "<%= worker_name %> Duration"
74
+ }
75
+ },
76
+ <% current_y += 3 %>
77
+
78
+
79
+
80
+
81
+ { <% "Processed Jobs By Worker" %>
82
+ "height": 3,
83
+ "width": 6,
84
+ "y": <%= current_y %>,
85
+ "x": 0,
86
+ "type": "metric",
87
+ "properties": {
88
+ "metrics": [
89
+ [ "${self:custom.funktor.DashboardNamespace}", "processed", "WorkerClassName", "<%= worker_name %>" ]
90
+ ],
91
+ "view": "timeSeries",
92
+ "stacked": false,
93
+ "region": "us-east-1",
94
+ "title": "<%= worker_name %> Processed Jobs",
95
+ "period": 60,
96
+ "stat": "Sum"
97
+ }
98
+ },
99
+ { <% "Failed Jobs By Worker" %>
100
+ "height": 3,
101
+ "width": 6,
102
+ "y": <%= current_y + 3 %>,
103
+ "x": 0,
104
+ "type": "metric",
105
+ "properties": {
106
+ "metrics": [
107
+ [ "${self:custom.funktor.DashboardNamespace}", "failed", "WorkerClassName", "<%= worker_name %>", { "color": "#d62728" }]
108
+ ],
109
+ "view": "timeSeries",
110
+ "stacked": false,
111
+ "region": "us-east-1",
112
+ "title": "<%= worker_name %> Failed Jobs",
113
+ "period": 60,
114
+ "stat": "Sum"
115
+ }
116
+ },
117
+ { <% "Job Duration By Worker" %>
118
+ "height": 6,
119
+ "width": 12,
120
+ "y": <%= current_y %>,
121
+ "x": 6,
122
+ "type": "metric",
123
+ "properties": {
124
+ "metrics": [
125
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ],
126
+ [ "...", { "stat": "p80" } ],
127
+ [ "...", { "stat": "p70" } ],
128
+ [ "...", { "stat": "p60" } ]
129
+ ],
130
+ "view": "timeSeries",
131
+ "stacked": false,
132
+ "region": "us-east-1",
133
+ "stat": "p90",
134
+ "period": 60,
135
+ "title": "<%= worker_name %> Duration Percentiles"
136
+ }
137
+ },
138
+ { <% "Maximum By Worker" %>
139
+ "height": 3,
140
+ "width": 6,
141
+ "y": <%= current_y %>,
142
+ "x": 18,
143
+ "type": "metric",
144
+ "properties": {
145
+ "metrics": [
146
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ]
147
+ ],
148
+ "view": "timeSeries",
149
+ "stacked": false,
150
+ "region": "us-east-1",
151
+ "stat": "Maximum",
152
+ "period": 60,
153
+ "title": "<%= worker_name %> Max"
154
+ }
155
+ },
156
+ { <% "Minimum By Worker" %>
157
+ "height": 3,
158
+ "width": 6,
159
+ "y": <%= current_y %>,
160
+ "x": 18,
161
+ "type": "metric",
162
+ "properties": {
163
+ "metrics": [
164
+ [ "${self:custom.funktor.DashboardNamespace}", "Duration", "WorkerClassName", "<%= worker_name %>" ]
165
+ ],
166
+ "view": "timeSeries",
167
+ "stacked": false,
168
+ "region": "us-east-1",
169
+ "stat": "Minimum",
170
+ "period": 60,
171
+ "title": "<%= worker_name %> Min"
172
+ }
173
+ },
174
+ <% current_y += 6 %>
175
+ <%- end -%>
176
+
177
+
24
178
  { <% "Job Duration By Worker" %>
25
179
  "height": 6,
26
180
  "width": 12,
@@ -65,6 +219,7 @@ Resources:
65
219
  }
66
220
  },
67
221
  <% current_y += 6 %>
222
+
68
223
  { <% "Job Duration By Queue" %>
69
224
  "height": 6,
70
225
  "width": 12,
@@ -147,6 +302,7 @@ Resources:
147
302
  "region": "us-east-1",
148
303
  "stat": "Sum",
149
304
  "period": 60,
305
+ "sparkline": true,
150
306
  "title": "Messages Per Minute"
151
307
  }
152
308
  },
@@ -167,6 +323,7 @@ Resources:
167
323
  "region": "us-east-1",
168
324
  "stat": "p10",
169
325
  "period": 60,
326
+ "sparkline": true,
170
327
  "title": "Handler Duration"
171
328
  }
172
329
  },
@@ -294,6 +451,7 @@ Resources:
294
451
  "region": "us-east-1",
295
452
  "stat": "Sum",
296
453
  "period": 60,
454
+ "sparkline": true,
297
455
  "title": "Messages Per Minute"
298
456
  }
299
457
  },
@@ -316,6 +474,7 @@ Resources:
316
474
  "region": "us-east-1",
317
475
  "stat": "p10",
318
476
  "period": 60,
477
+ "sparkline": true,
319
478
  "title": "Handler Duration"
320
479
  }
321
480
  },
@@ -444,6 +603,7 @@ Resources:
444
603
  "region": "us-east-1",
445
604
  "stat": "Average",
446
605
  "period": 60,
606
+ "sparkline": true,
447
607
  "title": "Messages To Be Scheduled"
448
608
  }
449
609
  },
@@ -464,6 +624,7 @@ Resources:
464
624
  "region": "us-east-1",
465
625
  "stat": "p10",
466
626
  "period": 60,
627
+ "sparkline": true,
467
628
  "title": "Handler Duration"
468
629
  }
469
630
  },
@@ -1,4 +1,4 @@
1
- use_docker: false
1
+ #use_docker: false
2
2
  #docker_yums:
3
3
  #- postgresql-devel
4
4
  #native_libs:
@@ -25,6 +25,16 @@ provider:
25
25
  runtime: <%= runtime %>
26
26
  stage: ${opt:stage, 'dev'}
27
27
  lambdaHashingVersion: 20201221
28
+ ecr:
29
+ scanOnPush: true
30
+ images:
31
+ funktorimage:
32
+ path: ./
33
+ file: Dockerfile
34
+ #platform: linux/amd64 # May be needed if you're on an M1.
35
+ buildArgs:
36
+ # Here you can specify ENV vars that you need at build time. For instance you may want to install private gems:
37
+ #BUNDLE_GEM__FURY__IO: ${env:BUNDLE_GEM__FURY__IO}
28
38
  environment: ${file(funktor_config/environment.yml)}
29
39
  versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
30
40
  logRetentionInDays: 7
@@ -39,9 +49,9 @@ custom:
39
49
  # commands. Or fallsback to what we have set in the provider section.
40
50
  stage: ${self:provider.stage, 'dev'}
41
51
  funktor: ${file(funktor_config/funktor.yml)}
42
- rubyLayer: ${file(funktor_config/ruby_layer.yml)}
52
+ #rubyLayer: ${file(funktor_config/ruby_layer.yml)}
43
53
 
44
- package: ${file(funktor_config/package.yml)}
54
+ #package: ${file(funktor_config/package.yml)}
45
55
 
46
56
  functions:
47
57
  <%- all_function_definitions.each do |function_definition| -%>
@@ -54,4 +64,4 @@ resources:
54
64
  <%- end -%>
55
65
 
56
66
  plugins:
57
- - serverless-ruby-layer
67
+ #- serverless-ruby-layer
@@ -18,11 +18,11 @@ module Funktor
18
18
  end
19
19
 
20
20
  def sqs_client
21
- @sqs_client ||= ::Aws::SQS::Client.new
21
+ Funktor.sqs_client
22
22
  end
23
23
 
24
24
  def dynamodb_client
25
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
25
+ Funktor.dynamodb_client
26
26
  end
27
27
 
28
28
  def dispatch(job)
@@ -30,11 +30,13 @@ module Funktor
30
30
  # TODO : This number should be configurable via ENV var
31
31
  if job.delay < 60 # for now we're testing with just one minute * 5 # 5 minutes
32
32
  Funktor.logger.debug "pushing to work queue for delay = #{job.delay}"
33
- # We push to the jobs table first becauase the work queue handler will expect to be able
34
- # to update the stats of a record that's already in the table.
35
- # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
36
- # work queue handler to be ok with a job that's not yet in the table?
37
- push_to_jobs_table(job, "queued")
33
+ if Funktor.enable_work_queue_visibility
34
+ # We push to the jobs table first becauase the work queue handler will expect to be able
35
+ # to update the stats of a record that's already in the table.
36
+ # TODO : For time sensitive jobs this is probably less than optimal. Can we update the
37
+ # work queue handler to be ok with a job that's not yet in the table?
38
+ push_to_jobs_table(job, "queued")
39
+ end
38
40
  push_to_work_queue(job)
39
41
  if job.is_retry?
40
42
  @tracker.track(:retryActivated, job)
@@ -9,11 +9,11 @@ module Funktor
9
9
  end
10
10
 
11
11
  def dynamodb_client
12
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
12
+ Funktor.dynamodb_client
13
13
  end
14
14
 
15
15
  def sqs_client
16
- @sqs_client ||= ::Aws::SQS::Client.new
16
+ Funktor.sqs_client
17
17
  end
18
18
 
19
19
  def delayed_job_table
@@ -61,23 +61,39 @@ module Funktor
61
61
  end
62
62
 
63
63
  def activate_job(job_shard, job_id, current_category, queue_immediately = false)
64
+ # TODO: WorkQueueVisibilityMiddleware to alter what happens here? Maybe we delete by default and then the middleware puts it back in the table?
64
65
  # First we conditionally update the item in Dynamo to be sure that another scheduler hasn't gotten
65
66
  # to it, and if that works then send to SQS. This is basically how Sidekiq scheduler works.
66
- response = dynamodb_client.update_item({
67
- key: {
68
- "jobShard" => job_shard,
69
- "jobId" => job_id
70
- },
71
- update_expression: "SET category = :category, queueable = :queueable",
72
- condition_expression: "category = :current_category",
73
- expression_attribute_values: {
74
- ":current_category" => current_category,
75
- ":queueable" => "false",
76
- ":category" => "queued"
77
- },
78
- table_name: delayed_job_table,
79
- return_values: "ALL_OLD"
80
- })
67
+ response = if Funktor.enable_work_queue_visibility
68
+ dynamodb_client.update_item({
69
+ key: {
70
+ "jobShard" => job_shard,
71
+ "jobId" => job_id
72
+ },
73
+ update_expression: "SET category = :category, queueable = :queueable",
74
+ condition_expression: "category = :current_category",
75
+ expression_attribute_values: {
76
+ ":current_category" => current_category,
77
+ ":queueable" => "false",
78
+ ":category" => "queued"
79
+ },
80
+ table_name: delayed_job_table,
81
+ return_values: "ALL_OLD"
82
+ })
83
+ else
84
+ dynamodb_client.delete_item({
85
+ key: {
86
+ "jobShard" => job_shard,
87
+ "jobId" => job_id
88
+ },
89
+ condition_expression: "category = :current_category",
90
+ expression_attribute_values: {
91
+ ":current_category" => current_category
92
+ },
93
+ table_name: delayed_job_table,
94
+ return_values: "ALL_OLD"
95
+ })
96
+ end
81
97
  if response.attributes # this means the record was still there in the state we expected
82
98
  Funktor.logger.debug "response.attributes ====== "
83
99
  Funktor.logger.debug response.attributes
@@ -19,7 +19,7 @@ module Funktor
19
19
  private
20
20
 
21
21
  def client
22
- @client ||= ::Aws::SQS::Client.new
22
+ Funktor.sqs_client
23
23
  end
24
24
 
25
25
  def queue_url(payload)
@@ -1,3 +1,3 @@
1
1
  module Funktor
2
- VERSION = "0.7.5"
2
+ VERSION = "0.7.8"
3
3
  end
@@ -128,7 +128,7 @@ module Funktor
128
128
  end
129
129
 
130
130
  def dynamodb_client
131
- @dynamodb_client ||= ::Aws::DynamoDB::Client.new
131
+ Funktor.dynamodb_client
132
132
  end
133
133
 
134
134
  # start the server if ruby file executed directly