lambdakiq 2.2.0 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/release.yml +24 -0
  3. data/.github/workflows/test.yml +15 -0
  4. data/.ruby-version +1 -0
  5. data/CHANGELOG.md +26 -1
  6. data/Gemfile.lock +150 -66
  7. data/README.md +38 -10
  8. data/bin/bootstrap +1 -1
  9. data/bin/console +1 -1
  10. data/bin/setup +1 -1
  11. data/bin/test +1 -1
  12. data/lambdakiq.gemspec +2 -2
  13. data/lib/lambdakiq/adapter.rb +7 -3
  14. data/lib/lambdakiq/error.rb +7 -0
  15. data/lib/lambdakiq/job.rb +10 -1
  16. data/lib/lambdakiq/message.rb +5 -1
  17. data/lib/lambdakiq/metrics.rb +1 -1
  18. data/lib/lambdakiq/railtie.rb +3 -1
  19. data/lib/lambdakiq/record.rb +4 -0
  20. data/lib/lambdakiq/version.rb +1 -1
  21. data/lib/lambdakiq/worker.rb +7 -0
  22. metadata +8 -9
  23. data/.github/workflows/cicd.yml +0 -17
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 84d147fdbb851da5e7e6ebace4cd2eb5a00ae53d3bd0d36981d9cb90c93bac5b
4
- data.tar.gz: 534d3f92cde56e7462202ebfd0b828039974d37792ee19d4c30b64ea0e901d61
3
+ metadata.gz: bca5d613a292ddbe8afe3c19fc74d060379d60c66265a2d60a197931fb82b824
4
+ data.tar.gz: ec92b80072bf1995246a9c5fbdac3c7ba27a11277065b7f44b4a65cb235eba6c
5
5
  SHA512:
6
- metadata.gz: 6cff69c86fae5f46a63cfbd7ee6290fb8aa472a6dc1377424df1cf8daf17ca96c373a87577c4f50ebaf081d8c9b13421e032f8b71b314cbd4135ec2ca7ca7dfe
7
- data.tar.gz: bed7c2407983c15b2ed48ba63362afe7b7a305c7415ec0deb533e25928ea805b8b5c8ad91bbeee4b51a7d3b69008e4854910b9093f392f454fdb78c8e8550db6
6
+ metadata.gz: 9933e96ad83da66e4101deec6247a5ecdceceec48210f106895ce43e7904670b173776e09641a5af0638fc82d0f2818e87b6e4b27f9fa8ff18d9fcad6d3a4ad4
7
+ data.tar.gz: b6a6eb0e82571852426a698c4d915fa01c19bb39366f90bbbb1435ba466cc4428acfe2f57c8eaa63f141db6579f6aa54b2e6273f304cd37d00de2decb5f5e370
data/.github/workflows/release.yml ADDED
@@ -0,0 +1,24 @@
1
+ name: Release Gem
2
+ on:
3
+ workflow_dispatch:
4
+ branches:
5
+ - master
6
+
7
+ jobs:
8
+ push:
9
+ name: Push gem to RubyGems.org
10
+ runs-on: ubuntu-latest
11
+
12
+ permissions:
13
+ id-token: write
14
+ contents: write
15
+
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - name: Set up Ruby
19
+ uses: ruby/setup-ruby@v1
20
+ with:
21
+ bundler-cache: true
22
+ ruby-version: "3.4.4"
23
+
24
+ - uses: rubygems/release-gem@v1
data/.github/workflows/test.yml ADDED
@@ -0,0 +1,15 @@
1
+ name: CI/CD
2
+ on: [push]
3
+ jobs:
4
+ test:
5
+ runs-on: ubuntu-24.04
6
+ steps:
7
+ - name: Checkout
8
+ uses: actions/checkout@v4
9
+ - name: Set up Ruby
10
+ uses: ruby/setup-ruby@v1
11
+ with:
12
+ bundler-cache: true
13
+ ruby-version: "3.4.4"
14
+ - name: Test
15
+ run: bundle exec rake test
data/.ruby-version ADDED
@@ -0,0 +1 @@
1
+ 3.4.4
data/CHANGELOG.md CHANGED
@@ -1,6 +1,31 @@
1
1
  # Keep A Changelog!
2
2
 
3
- See this http://keepachangelog.com link for information on how we want this documented formatted.
3
+ See this http://keepachangelog.com link for information on how we want this formatted.
4
+
5
+ ## v2.4.0
6
+
7
+ ### Added
8
+
9
+ - `message_group_id` option for `lambdakiq_options` to customize the FIFO message group id per job.
10
+
11
+ ### Fixed
12
+
13
+ - Preserve FIFO order within a batch: jobs behind a failed or delayed job of the same message group are skipped and returned to the queue.
14
+
15
+ ### Changed
16
+
17
+ - Update to Ruby 3.4.4 and Bundler 4.
18
+ - Update all gems, including Rails 6.1 → 8.1 and aws-sdk-sqs 1.35 → 1.116.
19
+ - Fix duplicate `JobName` key in CloudWatch metrics output (would raise an error in json 3.0).
20
+ - Modernize CI to use `ruby/setup-ruby` directly on `ubuntu-24.04`.
21
+ - Add `release.yml` workflow for trusted publishing to RubyGems.org.
22
+ - Fix error logging for failed `async: true` enqueues, which raised a `NameError` and was silently lost.
23
+
24
+ ## v2.3.0
25
+
26
+ ### Added
27
+
28
+ Support `enqueue_after_transaction_commit?` for rails 7.2.
4
29
 
5
30
  ## v2.2.0
6
31
 
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- lambdakiq (2.2.0)
4
+ lambdakiq (2.4.0)
5
5
  activejob
6
6
  aws-sdk-sqs
7
7
  concurrent-ruby
@@ -10,93 +10,177 @@ PATH
10
10
  GEM
11
11
  remote: https://rubygems.org/
12
12
  specs:
13
- actionpack (6.1.1)
14
- actionview (= 6.1.1)
15
- activesupport (= 6.1.1)
16
- rack (~> 2.0, >= 2.0.9)
13
+ actionpack (8.1.3)
14
+ actionview (= 8.1.3)
15
+ activesupport (= 8.1.3)
16
+ nokogiri (>= 1.8.5)
17
+ rack (>= 2.2.4)
18
+ rack-session (>= 1.0.1)
17
19
  rack-test (>= 0.6.3)
18
- rails-dom-testing (~> 2.0)
19
- rails-html-sanitizer (~> 1.0, >= 1.2.0)
20
- actionview (6.1.1)
21
- activesupport (= 6.1.1)
20
+ rails-dom-testing (~> 2.2)
21
+ rails-html-sanitizer (~> 1.6)
22
+ useragent (~> 0.16)
23
+ actionview (8.1.3)
24
+ activesupport (= 8.1.3)
22
25
  builder (~> 3.1)
23
- erubi (~> 1.4)
24
- rails-dom-testing (~> 2.0)
25
- rails-html-sanitizer (~> 1.1, >= 1.2.0)
26
- activejob (6.1.1)
27
- activesupport (= 6.1.1)
26
+ erubi (~> 1.11)
27
+ rails-dom-testing (~> 2.2)
28
+ rails-html-sanitizer (~> 1.6)
29
+ activejob (8.1.3)
30
+ activesupport (= 8.1.3)
28
31
  globalid (>= 0.3.6)
29
- activesupport (6.1.1)
30
- concurrent-ruby (~> 1.0, >= 1.0.2)
32
+ activesupport (8.1.3)
33
+ base64
34
+ bigdecimal
35
+ concurrent-ruby (~> 1.0, >= 1.3.1)
36
+ connection_pool (>= 2.2.5)
37
+ drb
31
38
  i18n (>= 1.6, < 2)
39
+ json
40
+ logger (>= 1.4.2)
32
41
  minitest (>= 5.1)
33
- tzinfo (~> 2.0)
34
- zeitwerk (~> 2.3)
35
- aws-eventstream (1.1.0)
36
- aws-partitions (1.422.0)
37
- aws-sdk-core (3.111.2)
42
+ securerandom (>= 0.3)
43
+ tzinfo (~> 2.0, >= 2.0.5)
44
+ uri (>= 0.13.1)
45
+ aws-eventstream (1.4.0)
46
+ aws-partitions (1.1257.0)
47
+ aws-sdk-core (3.251.0)
48
+ aws-eventstream (~> 1, >= 1.3.0)
49
+ aws-partitions (~> 1, >= 1.992.0)
50
+ aws-sigv4 (~> 1.9)
51
+ base64
52
+ bigdecimal
53
+ jmespath (~> 1, >= 1.6.1)
54
+ logger
55
+ aws-sdk-sqs (1.116.0)
56
+ aws-sdk-core (~> 3, >= 3.248.0)
57
+ aws-sigv4 (~> 1.5)
58
+ aws-sigv4 (1.12.1)
38
59
  aws-eventstream (~> 1, >= 1.0.2)
39
- aws-partitions (~> 1, >= 1.239.0)
40
- aws-sigv4 (~> 1.1)
41
- jmespath (~> 1.0)
42
- aws-sdk-sqs (1.35.0)
43
- aws-sdk-core (~> 3, >= 3.109.0)
44
- aws-sigv4 (~> 1.1)
45
- aws-sigv4 (1.2.2)
46
- aws-eventstream (~> 1, >= 1.0.2)
47
- builder (3.2.4)
60
+ base64 (0.3.0)
61
+ bigdecimal (4.1.2)
62
+ builder (3.3.0)
48
63
  coderay (1.1.3)
49
- concurrent-ruby (1.1.8)
64
+ concurrent-ruby (1.3.6)
65
+ connection_pool (3.0.2)
50
66
  crass (1.0.6)
51
- erubi (1.10.0)
52
- globalid (0.4.2)
53
- activesupport (>= 4.2.0)
54
- i18n (1.8.7)
67
+ date (3.5.1)
68
+ drb (2.2.3)
69
+ erb (6.0.4)
70
+ erubi (1.13.1)
71
+ globalid (1.3.0)
72
+ activesupport (>= 6.1)
73
+ i18n (1.14.8)
55
74
  concurrent-ruby (~> 1.0)
56
- jmespath (1.4.0)
57
- loofah (2.9.0)
75
+ io-console (0.8.2)
76
+ irb (1.18.0)
77
+ pp (>= 0.6.0)
78
+ prism (>= 1.3.0)
79
+ rdoc (>= 4.0.0)
80
+ reline (>= 0.4.2)
81
+ jmespath (1.6.2)
82
+ json (2.19.8)
83
+ logger (1.7.0)
84
+ loofah (2.25.1)
58
85
  crass (~> 1.0.2)
59
- nokogiri (>= 1.5.9)
86
+ nokogiri (>= 1.12.0)
60
87
  macaddr (1.7.2)
61
88
  systemu (~> 2.6.5)
62
- method_source (1.0.0)
63
- mini_portile2 (2.5.0)
64
- minitest (5.14.2)
65
- minitest-focus (1.2.1)
66
- minitest (>= 4, < 6)
67
- mocha (1.11.2)
68
- nokogiri (1.11.1)
69
- mini_portile2 (~> 2.5.0)
89
+ method_source (1.1.0)
90
+ mini_portile2 (2.8.9)
91
+ minitest (6.0.6)
92
+ drb (~> 2.0)
93
+ prism (~> 1.5)
94
+ minitest-focus (1.4.1)
95
+ minitest (> 5.0)
96
+ mocha (3.1.0)
97
+ ruby2_keywords (>= 0.0.5)
98
+ nokogiri (1.19.3)
99
+ mini_portile2 (~> 2.8.2)
100
+ racc (~> 1.4)
101
+ nokogiri (1.19.3-aarch64-linux-gnu)
102
+ racc (~> 1.4)
103
+ nokogiri (1.19.3-aarch64-linux-musl)
104
+ racc (~> 1.4)
105
+ nokogiri (1.19.3-arm-linux-gnu)
106
+ racc (~> 1.4)
107
+ nokogiri (1.19.3-arm-linux-musl)
108
+ racc (~> 1.4)
109
+ nokogiri (1.19.3-arm64-darwin)
110
+ racc (~> 1.4)
111
+ nokogiri (1.19.3-x86_64-darwin)
112
+ racc (~> 1.4)
113
+ nokogiri (1.19.3-x86_64-linux-gnu)
114
+ racc (~> 1.4)
115
+ nokogiri (1.19.3-x86_64-linux-musl)
70
116
  racc (~> 1.4)
71
- pry (0.13.1)
117
+ pp (0.6.3)
118
+ prettyprint
119
+ prettyprint (0.2.0)
120
+ prism (1.9.0)
121
+ pry (0.16.0)
72
122
  coderay (~> 1.1)
73
123
  method_source (~> 1.0)
74
- racc (1.5.2)
75
- rack (2.2.3)
76
- rack-test (1.1.0)
77
- rack (>= 1.0, < 3)
78
- rails-dom-testing (2.0.3)
79
- activesupport (>= 4.2.0)
124
+ reline (>= 0.6.0)
125
+ psych (5.4.0)
126
+ date
127
+ stringio
128
+ racc (1.8.1)
129
+ rack (3.2.6)
130
+ rack-session (2.1.2)
131
+ base64 (>= 0.1.0)
132
+ rack (>= 3.0.0)
133
+ rack-test (2.2.0)
134
+ rack (>= 1.3)
135
+ rackup (2.3.1)
136
+ rack (>= 3)
137
+ rails-dom-testing (2.3.0)
138
+ activesupport (>= 5.0.0)
139
+ minitest
80
140
  nokogiri (>= 1.6)
81
- rails-html-sanitizer (1.3.0)
82
- loofah (~> 2.3)
83
- railties (6.1.1)
84
- actionpack (= 6.1.1)
85
- activesupport (= 6.1.1)
86
- method_source
87
- rake (>= 0.8.7)
88
- thor (~> 1.0)
89
- rake (13.0.1)
141
+ rails-html-sanitizer (1.7.0)
142
+ loofah (~> 2.25)
143
+ nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
144
+ railties (8.1.3)
145
+ actionpack (= 8.1.3)
146
+ activesupport (= 8.1.3)
147
+ irb (~> 1.13)
148
+ rackup (>= 1.0.0)
149
+ rake (>= 12.2)
150
+ thor (~> 1.0, >= 1.2.2)
151
+ tsort (>= 0.2)
152
+ zeitwerk (~> 2.6)
153
+ rake (13.4.2)
154
+ rdoc (7.2.0)
155
+ erb
156
+ psych (>= 4.0.0)
157
+ tsort
158
+ reline (0.6.3)
159
+ io-console (~> 0.5)
160
+ ruby2_keywords (0.0.5)
161
+ securerandom (0.4.1)
162
+ stringio (3.2.0)
90
163
  systemu (2.6.5)
91
- thor (1.1.0)
92
- tzinfo (2.0.4)
164
+ thor (1.5.0)
165
+ tsort (0.2.0)
166
+ tzinfo (2.0.6)
93
167
  concurrent-ruby (~> 1.0)
168
+ uri (1.1.1)
169
+ useragent (0.16.11)
94
170
  uuid (2.3.9)
95
171
  macaddr (~> 1.0)
96
- zeitwerk (2.4.2)
172
+ zeitwerk (2.8.2)
97
173
 
98
174
  PLATFORMS
175
+ aarch64-linux-gnu
176
+ aarch64-linux-musl
177
+ arm-linux-gnu
178
+ arm-linux-musl
179
+ arm64-darwin
99
180
  ruby
181
+ x86_64-darwin
182
+ x86_64-linux-gnu
183
+ x86_64-linux-musl
100
184
 
101
185
  DEPENDENCIES
102
186
  bundler
@@ -109,4 +193,4 @@ DEPENDENCIES
109
193
  uuid
110
194
 
111
195
  BUNDLED WITH
112
- 2.1.4
196
+ 4.0.13
data/README.md CHANGED
@@ -1,10 +1,11 @@
1
1
  ![Lambdakiq: ActiveJob on SQS & Lambda](images/Lambdakiq.png)
2
2
 
3
- ![Test](https://github.com/customink/lambdakiq/workflows/Test/badge.svg)
3
+ ![Test](https://github.com/rails-lambda/lambdakiq/workflows/Test/badge.svg)
4
4
 
5
5
  # Lambdakiq
6
6
 
7
- <a href="https://lamby.custominktech.com"><img src="https://user-images.githubusercontent.com/2381/59363668-89edeb80-8d03-11e9-9985-2ce14361b7e3.png" alt="Lamby: Simple Rails & AWS Lambda Integration using Rack." align="right" width="300" /></a>A drop-in replacement for [Sidekiq](https://github.com/mperham/sidekiq) when running Rails in AWS Lambda using the [Lamby](https://lamby.custominktech.com) gem.
7
+ <a href="https://lamby.cloud"><img src="https://raw.githubusercontent.com/rails-lambda/lamby/master/images/social2.png" alt="Lamby: Simple Rails & AWS Lambda Integration using Rack." align="right" width="450" style="margin-left:1rem;margin-bottom:1rem;" /></a>
8
+ A drop-in replacement for [Sidekiq](https://github.com/mperham/sidekiq) when running Rails in AWS Lambda using the [Lamby](https://lamby.cloud) gem.
8
9
 
9
10
  Lambdakiq allows you to leverage AWS' managed infrastructure to the fullest extent. Gone are the days of managing pods and long polling processes. Instead AWS delivers messages directly to your Rails' job functions and scales it up and down as needed. Observability is built in using AWS CloudWatch Metrics, Dashboards, and Alarms. Learn more about [Using AWS Lambda with Amazon SQS](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) or get started now.
10
11
 
@@ -20,7 +21,7 @@ Lambdakiq allows you to leverage AWS' managed infrastructure to the fullest exte
20
21
 
21
22
  ## Project Setup
22
23
 
23
- This gem assumes your Rails application is on AWS Lambda, ideally with our [Lamby](https://lamby.custominktech.com) gem. It could be using Lambda's traditional zip package type or the newer [container](https://dev.to/aws-heroes/lambda-containers-with-rails-a-perfect-match-4lgb) format. If Rails on Lambda is new to you, consider following our [quick start](https://lamby.custominktech.com/docs/quick_start) guide to get your first application up and running. From there, to use Lambdakiq, here are steps to setup your project
24
+ This gem assumes your Rails application is on AWS Lambda, ideally with our [Lamby](https://lamby.cloud) gem. It could be using Lambda's traditional zip package type or the newer [container](https://dev.to/aws-heroes/lambda-containers-with-rails-a-perfect-match-4lgb) format. If Rails on Lambda is new to you, consider following our [quick start](https://lamby.cloud/docs/quick-start) guide to get your first application up and running. From there, to use Lambdakiq, here are steps to setup your project
24
25
 
25
26
  ### Bundle & Config
26
27
 
@@ -56,11 +57,11 @@ ActionMailer::MailDeliveryJob.include Lambdakiq::Worker
56
57
  ActionMailer::MailDeliveryJob.queue_as ENV['JOBS_QUEUE_NAME']
57
58
  ```
58
59
 
59
- The same Docker image will be used for both your `web` and `jobs` functions (example setup in following sections). The [Lamby](https://lamby.custominktech.com) gem can automatically can detect if Lambdakiq is present when using the newer `Lamby.cmd` or older lower `Lamby.handler` method. That said, please take a look at the `JobsLambda` in the following section and how `ImageConfig` is used as the golden path for sharing containers.
60
+ The same Docker image will be used for both your `web` and `jobs` functions (example setup in following sections). The [Lamby](https://lamby.cloud) gem can automatically can detect if Lambdakiq is present when using the newer `Lamby.cmd` or older lower `Lamby.handler` method. That said, please take a look at the `JobsLambda` in the following section and how `ImageConfig` is used as the golden path for sharing containers.
60
61
 
61
62
  ### SQS Resources
62
63
 
63
- Open up your project's SAM [`template.yaml`](https://lamby.custominktech.com/docs/anatomy#file-template-yaml) file and make the following additions and changes. First, we need to create your [SQS queues](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html) under the `Resources` section.
64
+ Open up your project's SAM [`template.yaml`](https://lamby.cloud/docs/anatomy#file-template-yaml) file and make the following additions and changes. First, we need to create your [SQS queues](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html) under the `Resources` section.
64
65
 
65
66
  ```yaml
66
67
  JobsQueue:
@@ -168,8 +169,26 @@ config.lambdakiq
168
169
  ```
169
170
 
170
171
  - `max_retries=` - Retries for all jobs. Default is the Lambdakiq maximum of `12`.
172
+ - `metrics_enabled=` - Set to `false` to stop emitting CloudWatch Embedded Metrics. Default is `true`.
171
173
  - `metrics_namespace=` - The CloudWatch Embedded Metrics namespace. Default is `Lambdakiq`.
172
- - `metrics_logger=` - Set to the Rails logger which is STDOUT via Lamby/Lambda.
174
+ - `metrics_app_name=` - The `AppName` metric dimension. Defaults to your application's module name.
175
+ - `metrics_logger=` - Where metrics are written. Defaults to the Rails logger, which is STDOUT via Lamby/Lambda.
176
+
177
+ #### Disabling CloudWatch Metrics
178
+
179
+ Lambdakiq emits [CloudWatch Embedded Metrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Embedded_Metric_Format.html) for every ActiveJob event, dimensioned by `AppName`, `JobEvent`, and `JobName`. Because custom metrics are billed per unique dimension combination, applications with many job classes or very high job volume can see meaningful CloudWatch cost. Metric properties also include each job's arguments, which you may not want in your logs.
180
+
181
+ Either turn metrics off entirely:
182
+
183
+ ```ruby
184
+ config.lambdakiq.metrics_enabled = false
185
+ ```
186
+
187
+ Or keep them, but send them somewhere other than STDOUT so CloudWatch does not ingest them:
188
+
189
+ ```ruby
190
+ config.lambdakiq.metrics_logger = Logger.new(File::NULL)
191
+ ```
173
192
 
174
193
  ### ActiveJob Configs
175
194
 
@@ -182,6 +201,15 @@ end
182
201
  ```
183
202
 
184
203
  - `retry` - Overrides the default Lambdakiq `max_retries` for this one job.
204
+ - `message_group_id` - FIFO queues only. A string, or a proc called with the job, used as the message group id instead of the unique job id. Jobs sharing a group id are processed one at a time, in order. The value is converted to a string, blank values fall back to the job id, and values over 128 characters are replaced by their SHA256 digest.
205
+
206
+ ```ruby
207
+ class SyncOrderJob < ApplicationJob
208
+ lambdakiq_options message_group_id: ->(job) { "SyncOrderJob-#{job.arguments.first}" }
209
+ end
210
+ ```
211
+
212
+ Messages in a group wait for the one in flight, so a job delayed with `wait` or waiting on a retry holds up the rest of its group. Within a batch, jobs behind a failed or delayed job of the same group are not performed and are returned to the queue.
185
213
 
186
214
  ## Observability with CloudWatch
187
215
 
@@ -235,7 +263,7 @@ And finally, here are the metrics which each dimension can chart using [CloudWat
235
263
 
236
264
  Please share how you are using CloudWatch to monitor and/or alert on your ActiveJobs with Lambdakiq!
237
265
 
238
- 💬 https://github.com/customink/lambdakiq/discussions/3
266
+ 💬 https://github.com/rails-lambda/lambdakiq/discussions/3
239
267
 
240
268
  ## Common Questions
241
269
 
@@ -244,7 +272,7 @@ Please share how you are using CloudWatch to monitor and/or alert on your Active
244
272
  **Are FIFO Queues Supported?** - Yes. When you create your [AWS::SQS::Queue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html) resources you can set the [FifoQueue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html#aws-sqs-queue-fifoqueue) property to `true`. Remember that both your jobs queue and the redrive queue must be the same. When using FIFO we:
245
273
 
246
274
  - Simulate `delay_seconds` for ActiveJob's wait by using visibility timeouts under the hood. We still cap it to non-FIFO's 15 minutes.
247
- - Set both the messages `message_group_id` and `message_deduplication_id` to the unique job id provided by ActiveJob.
275
+ - Set both the messages `message_group_id` and `message_deduplication_id` to the unique job id provided by ActiveJob. The group id can be customized per job with the `message_group_id` option.
248
276
 
249
277
  **Can I Use Multiple Queues?** - Yes. Nothing is stopping you from creating any number of queues and/or functions to process them. Your subclasses can use ActiveJob's `queue_as` method as needed. This is an easy way to handle job priorities too.
250
278
 
@@ -266,8 +294,8 @@ $ ./bin/setup
266
294
  $ ./bin/test
267
295
  ```
268
296
 
269
- Bug reports and pull requests are welcome on GitHub at https://github.com/customink/lambdakiq. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
297
+ Bug reports and pull requests are welcome on GitHub at https://github.com/rails-lambda/lambdakiq. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
270
298
 
271
299
  ## Code of Conduct
272
300
 
273
- Everyone interacting in the Lambdakiq project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/customink/lambdakiq/blob/master/CODE_OF_CONDUCT.md).
301
+ Everyone interacting in the Lambdakiq project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/rails-lambda/lambdakiq/blob/master/CODE_OF_CONDUCT.md).
data/bin/bootstrap CHANGED
@@ -3,4 +3,4 @@ set -e
3
3
 
4
4
  echo '== Building containers =='
5
5
  docker pull lambci/lambda:build-ruby2.7
6
- docker-compose build
6
+ docker compose build
data/bin/console CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/bin/bash
2
2
  set -e
3
3
 
4
- docker-compose run \
4
+ docker compose run \
5
5
  lambdakiqgem \
6
6
  ./bin/_console
data/bin/setup CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/bin/bash
2
2
  set -e
3
3
 
4
- docker-compose run \
4
+ docker compose run \
5
5
  lambdakiqgem \
6
6
  ./bin/_setup
data/bin/test CHANGED
@@ -3,6 +3,6 @@ set -e
3
3
 
4
4
  export RAILS_ENV="test"
5
5
 
6
- docker-compose run \
6
+ docker compose run \
7
7
  lambdakiqgem \
8
8
  ./bin/_test
data/lambdakiq.gemspec CHANGED
@@ -6,10 +6,10 @@ Gem::Specification.new do |spec|
6
6
  spec.name = "lambdakiq"
7
7
  spec.version = Lambdakiq::VERSION
8
8
  spec.authors = ["Ken Collins"]
9
- spec.email = ["kcollins@customink.com"]
9
+ spec.email = ["ken@metaskills.net"]
10
10
  spec.summary = %q{Scalable Rails Background Processing with AWS Lambda & SQS.}
11
11
  spec.description = %q{Scalable Rails Background Processing with AWS Lambda & SQS.}
12
- spec.homepage = "https://github.com/customink/lambdakiq"
12
+ spec.homepage = "https://github.com/rails-lambda/lambdakiq"
13
13
  spec.license = "MIT"
14
14
  spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
15
15
  `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|images)/}) }
data/lib/lambdakiq/adapter.rb CHANGED
@@ -10,6 +10,10 @@ module ActiveJob
10
10
  enqueue job, delay_seconds: delay_seconds(timestamp)
11
11
  end
12
12
 
13
+ def enqueue_after_transaction_commit?
14
+ true
15
+ end
16
+
13
17
  private
14
18
 
15
19
  def delay_seconds(timestamp)
@@ -25,11 +29,11 @@ module ActiveJob
25
29
  def _enqueue_async(job, options = {})
26
30
  Concurrent::Promise
27
31
  .execute { _enqueue(job, options) }
28
- .on_error { |e| async_enqueue_error(e) }
32
+ .on_error { |e| async_enqueue_error(job, e) }
29
33
  end
30
34
 
31
- def async_enqueue_error(e)
32
- msg = "[Lambdakiq] Failed to queue job #{job}. Reason: #{e}"
35
+ def async_enqueue_error(job, e)
36
+ msg = "[Lambdakiq] Failed to queue job #{job.class.name} (#{job.job_id}). Reason: #{e}"
33
37
  Rails.logger.error(msg)
34
38
  end
35
39
 
data/lib/lambdakiq/error.rb CHANGED
@@ -8,4 +8,11 @@ module Lambdakiq
8
8
  set_backtrace([])
9
9
  end
10
10
  end
11
+
12
+ class FifoGroupError < Error
13
+ def initialize(error)
14
+ super
15
+ set_backtrace([])
16
+ end
17
+ end
11
18
  end
data/lib/lambdakiq/job.rb CHANGED
@@ -8,7 +8,12 @@ module Lambdakiq
8
8
  def handler(event)
9
9
  records = Event.records(event)
10
10
  jobs = records.map { |record| new(record) }
11
- jobs.each(&:perform)
11
+ failed_groups = []
12
+ jobs.each do |job|
13
+ group_id = job.record.fifo_message_group_id
14
+ failed_groups.include?(group_id) ? job.skip : job.perform
15
+ failed_groups << group_id if job.error && group_id
16
+ end
12
17
  failed_jobs = jobs.select { |j| j.error }
13
18
  item_failures = failed_jobs.map { |j| { itemIdentifier: j.provider_job_id } }
14
19
  { batchItemFailures: item_failures }
@@ -52,6 +57,10 @@ module Lambdakiq
52
57
  execute
53
58
  end
54
59
 
60
+ def skip
61
+ @error = FifoGroupError.new(active_job.job_id)
62
+ end
63
+
55
64
  def execute
56
65
  ActiveJob::Base.execute(job_data)
57
66
  delete_message
data/lib/lambdakiq/message.rb CHANGED
@@ -36,13 +36,17 @@ module Lambdakiq
36
36
 
37
37
  def message_params_fifo
38
38
  if queue.fifo?
39
- { message_group_id: job.job_id,
39
+ { message_group_id: message_group_id,
40
40
  message_deduplication_id: job.job_id }
41
41
  else
42
42
  {}
43
43
  end
44
44
  end
45
45
 
46
+ def message_group_id
47
+ job.try(:lambdakiq_message_group_id) || job.job_id
48
+ end
49
+
46
50
  def message_attributes
47
51
  LAMBDAKIQ_ATTRIBUTE.merge(delay_seconds_attribute)
48
52
  end
data/lib/lambdakiq/metrics.rb CHANGED
@@ -4,6 +4,7 @@ module Lambdakiq
4
4
 
5
5
  class << self
6
6
  def log(event)
7
+ return unless Lambdakiq.config.metrics_enabled
7
8
  new(event).log
8
9
  end
9
10
  end
@@ -67,7 +68,6 @@ module Lambdakiq
67
68
  put_metric 'Count', 1, 'Count'
68
69
  put_metric 'ExceptionCount', 1, 'Count' if exception_name
69
70
  set_property 'JobId', job.job_id
70
- set_property 'JobName', job_name
71
71
  set_property 'QueueName', job.queue_name
72
72
  set_property 'MessageId', job.provider_job_id if job.provider_job_id
73
73
  set_property 'ExceptionName', exception_name if exception_name
data/lib/lambdakiq/railtie.rb CHANGED
@@ -3,14 +3,16 @@ module Lambdakiq
3
3
  config.lambdakiq = ActiveSupport::OrderedOptions.new
4
4
  config.lambdakiq.max_retries = 12
5
5
  config.lambdakiq.metrics_namespace = 'Lambdakiq'
6
+ config.lambdakiq.metrics_enabled = true
6
7
 
7
8
  config.after_initialize do
8
9
  config.active_job.logger = Rails.logger
9
- config.lambdakiq.metrics_logger = Rails.logger
10
+ config.lambdakiq.metrics_logger ||= Rails.logger
10
11
  end
11
12
 
12
13
  initializer "lambdakiq.metrics" do |app|
13
14
  ActiveSupport::Notifications.subscribe(/active_job/) do |*args|
15
+ next unless Lambdakiq.config.metrics_enabled
14
16
  event = ActiveSupport::Notifications::Event.new *args
15
17
  Lambdakiq::Metrics.log(event)
16
18
  end
data/lib/lambdakiq/record.rb CHANGED
@@ -27,6 +27,10 @@ module Lambdakiq
27
27
  data['attributes']
28
28
  end
29
29
 
30
+ def fifo_message_group_id
31
+ attributes['MessageGroupId'] if queue_name.end_with?('.fifo')
32
+ end
33
+
30
34
  def fifo_delay_visibility_timeout
31
35
  fifo_delay_seconds - (Time.current - sent_at).to_i
32
36
  end
data/lib/lambdakiq/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Lambdakiq
2
- VERSION = '2.2.0'
2
+ VERSION = '2.4.0'
3
3
  end
data/lib/lambdakiq/worker.rb CHANGED
@@ -28,5 +28,12 @@ module Lambdakiq
28
28
  !!lambdakiq_options_hash[:async]
29
29
  end
30
30
 
31
+ def lambdakiq_message_group_id
32
+ group_id = lambdakiq_options_hash[:message_group_id]
33
+ group_id = group_id.call(self) if group_id.respond_to?(:call)
34
+ group_id = group_id.to_s
35
+ group_id.length > 128 ? Digest::SHA256.hexdigest(group_id) : group_id.presence
36
+ end
37
+
31
38
  end
32
39
  end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: lambdakiq
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 2.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ken Collins
8
- autorequire:
9
8
  bindir: exe
10
9
  cert_chain: []
11
- date: 2022-12-19 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: activejob
@@ -166,13 +165,15 @@ dependencies:
166
165
  version: '0'
167
166
  description: Scalable Rails Background Processing with AWS Lambda & SQS.
168
167
  email:
169
- - kcollins@customink.com
168
+ - ken@metaskills.net
170
169
  executables: []
171
170
  extensions: []
172
171
  extra_rdoc_files: []
173
172
  files:
174
- - ".github/workflows/cicd.yml"
173
+ - ".github/workflows/release.yml"
174
+ - ".github/workflows/test.yml"
175
175
  - ".gitignore"
176
+ - ".ruby-version"
176
177
  - CHANGELOG.md
177
178
  - CODE_OF_CONDUCT.md
178
179
  - Dockerfile
@@ -206,11 +207,10 @@ files:
206
207
  - lib/lambdakiq/version.rb
207
208
  - lib/lambdakiq/worker.rb
208
209
  - vendor/.keep
209
- homepage: https://github.com/customink/lambdakiq
210
+ homepage: https://github.com/rails-lambda/lambdakiq
210
211
  licenses:
211
212
  - MIT
212
213
  metadata: {}
213
- post_install_message:
214
214
  rdoc_options: []
215
215
  require_paths:
216
216
  - lib
@@ -225,8 +225,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
225
225
  - !ruby/object:Gem::Version
226
226
  version: '0'
227
227
  requirements: []
228
- rubygems_version: 3.1.2
229
- signing_key:
228
+ rubygems_version: 3.6.7
230
229
  specification_version: 4
231
230
  summary: Scalable Rails Background Processing with AWS Lambda & SQS.
232
231
  test_files: []
data/.github/workflows/cicd.yml DELETED
@@ -1,17 +0,0 @@
1
- name: Test
2
- on: [push]
3
- jobs:
4
- test:
5
- runs-on: ubuntu-latest
6
- steps:
7
- - name: Checkout
8
- uses: actions/checkout@v1
9
- - name: Bootstrap
10
- run: |
11
- ./bin/bootstrap
12
- - name: Setup
13
- run: |
14
- ./bin/setup
15
- - name: Test
16
- run: |
17
- ./bin/test