lambdakiq 1.0.1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c2216c121ae2ba5e013c8852bc71d8e3e84342cd67053e4d9933a88562c543a
-  data.tar.gz: f1c48212d2f14307fbb7a09ac10748d108b0d705270df8e3a5b54047bdcee758
+  metadata.gz: 94804599e9c2e4f8f701c34cee61a7d18db8f9a20fee85a3e24681e2bd555076
+  data.tar.gz: 7550236dacfc649a36468bbc4b3a7116a03b90d5943df4cdb0c08f7521ad505f
 SHA512:
-  metadata.gz: 687c1792d5f9e15ce30db4226a0248b46a874e2c8b55e81327a871977ccbff375ec67eb3c8279a2ff565e7e1c9f05f5147f70a0bec1e6308e8a8fbc2863440f3
-  data.tar.gz: 0e88a280f225db1dfb8251b0622e99bf3d781dba6c817fe8f70b8345725613421578da31cc5bac37d8664fd0390431518a741f92d71e8da4087506c742719e10
+  metadata.gz: e7592aaa326a0ed6fa6202c0877b61ed67f00d156078d37840dda13ee4a455ff4de1ac35a377dd95230288fdc6ae054b974ef990190c0a46b59a45eb81c10ee5
+  data.tar.gz: 2f8efaa743a20710fd57ad2aa1b0cf8983828100b6ec5880f0815320f186216a5420e362d24b1b21b4e0ae78187c75e517afc4e27ddf5f9d3c3b295803101c16

data/CHANGELOG.md

@@ -1,10 +1,21 @@
-
 # Keep A Changelog!
 
 See this http://keepachangelog.com link for information on how we want this documented formatted.
 
+## v2.0.0
+
+#### Changed
+
+- Leverage new `ReportBatchItemFailures` feature of SQS.
+
+## v1.0.2, v1.0.3, v1.0.4
+
+#### Fixed
+
+- Rails v5.2 compatibility. Metrics logging is are safe for non-Lambdakiq jobs
+
 ## v1.0.0
 
 #### Added
 
-* Initial release.
+- Initial release.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    lambdakiq (1.0.1)
+    lambdakiq (2.0.0)
       activejob
       aws-sdk-sqs
       concurrent-ruby

data/README.md

@@ -1,7 +1,7 @@
-![Test](https://github.com/customink/lambdakiq/workflows/Test/badge.svg)
-
 ![Lambdakiq: ActiveJob on SQS & Lambda](images/Lambdakiq.png)
 
+![Test](https://github.com/customink/lambdakiq/workflows/Test/badge.svg)
+
 # Lambdakiq
 
 <a href="https://lamby.custominktech.com"><img src="https://user-images.githubusercontent.com/2381/59363668-89edeb80-8d03-11e9-9985-2ce14361b7e3.png" alt="Lamby: Simple Rails & AWS Lambda Integration using Rack." align="right" width="300" /></a>A drop-in replacement for [Sidekiq](https://github.com/mperham/sidekiq) when running Rails in AWS Lambda using the [Lamby](https://lamby.custominktech.com) gem.
@@ -10,19 +10,18 @@ Lambdakiq allows you to leverage AWS' managed infrastructure to the fullest exte
 
 ## Key Features
 
-* Distinct web & jobs Lambda functions.
-* AWS fully managed polling. Event-driven.
-* Maximum 12 retries. Per job configurable.
-* Mirror Sidekiq's retry [backoff](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry) timing.
-* Last retry is at 11 hours 30 minutes.
-* Supports ActiveJob's wait/delay. Up to 15 minutes.
-* Dead messages are stored for up to 14 days.
+- Distinct web & jobs Lambda functions.
+- AWS fully managed polling. Event-driven.
+- Maximum 12 retries. Per job configurable.
+- Mirror Sidekiq's retry [backoff](https://github.com/mperham/sidekiq/wiki/Error-Handling#automatic-job-retry) timing.
+- Last retry is at 11 hours 30 minutes.
+- Supports ActiveJob's wait/delay. Up to 15 minutes.
+- Dead messages are stored for up to 14 days.
 
 ## Project Setup
 
 This gem assumes your Rails application is on AWS Lambda, ideally with our [Lamby](https://lamby.custominktech.com) gem. It could be using Lambda's traditional zip package type or the newer [container](https://dev.to/aws-heroes/lambda-containers-with-rails-a-perfect-match-4lgb) format. If Rails on Lambda is new to you, consider following our [quick start](https://lamby.custominktech.com/docs/quick_start) guide to get your first application up and running. From there, to use Lambdakiq, here are steps to setup your project
 
-
 ### Bundle & Config
 
 Add the Lambdakiq gem to your `Gemfile`.
@@ -31,7 +30,7 @@ Add the Lambdakiq gem to your `Gemfile`.
 gem 'lambdakiq'
 ```
 
-Open `config/initializers/production.rb` and set Lambdakiq as your ActiveJob queue adapter.
+Open `config/environments/production.rb` and set Lambdakiq as your ActiveJob queue adapter.
 
 ```ruby
 config.active_job.queue_adapter = :lambdakiq
@@ -46,6 +45,33 @@ class ApplicationJob < ActiveJob::Base
 end
 ```
 
+Using ActionMailer's built-in deliver job with ActiveJob? Make sure to include the Lambdakiq worker and set the queue name depending on your Rails version. You can do this in a newly created `config/initializers//action_mailer.rb` or another initializer of your choice.
+
+```ruby
+# Rails 5.x
+ActionMailer::DeliveryJob.include Lambdakiq::Worker
+ActionMailer::DeliveryJob.queue_as ENV['JOBS_QUEUE_NAME']
+# Rails 6.x
+ActionMailer::MailDeliveryJob.include Lambdakiq::Worker
+ActionMailer::MailDeliveryJob.queue_as ENV['JOBS_QUEUE_NAME']
+```
+
+The same Docker image will be used for both your `web` and `jobs` functions (example setup in following sections) which means the same `app.rb` handler would be used. The [Lamby](https://lamby.custominktech.com) gem automatically detects if Lambdakiq is being used so the following handler works as is.
+
+```ruby
+def handler(event:, context:)
+  Lamby.handler $app, event, context
+end
+```
+
+You can use the Lambdakiq handler directly in cases where your handler is a different method. Likewise there is a `Lambdakiq.jobs?(event)` helper function which returns true if the `messageAttributes` has a `lambdakiq` attribute.
+
+```ruby
+def jobs_handler(event:, context:)
+  Lambdakiq.handler(event)
+end
+```
+
 ### SQS Resources
 
 Open up your project's SAM [`template.yaml`](https://lamby.custominktech.com/docs/anatomy#file-template-yaml) file and make the following additions and changes. First, we need to create your [SQS queues](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html) under the `Resources` section.
@@ -91,7 +117,7 @@ Both functions will need capabilities to access the SQS jobs queue. We can add o
 
 ```yaml
 Policies:
-  - Version: '2012-10-17'
+  - Version: "2012-10-17"
     Statement:
       - Effect: Allow
         Action:
@@ -100,6 +126,8 @@ Policies:
           - !Sub arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:${JobsQueue.QueueName}
 ```
 
+### Overview
+
 Now we can duplicate our `RailsLambda` resource YAML (except for the `Events` property) to a new `JobsLambda` one. This gives us a distinct Lambda function to process jobs whose events, memory, timeout, and more can be independently tuned. However, both the `web` and `jobs` functions will use the same ECR container image!
 
 ```yaml
@@ -116,10 +144,12 @@ JobsLambda:
         Properties:
           Queue: !GetAtt JobsQueue.Arn
           BatchSize: 1
+          FunctionResponseTypes:
+            - ReportBatchItemFailures
     MemorySize: 1792
     PackageType: Image
     Policies:
-      - Version: '2012-10-17'
+      - Version: "2012-10-17"
         Statement:
           - Effect: Allow
             Action:
@@ -131,10 +161,10 @@ JobsLambda:
 
 Here are some key aspects of our `JobsLambda` resource above:
 
-* The `Events` property uses the [SQS Type](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-sqs.html).
-* Our [BatchSize](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-sqs.html#sam-function-sqs-batchsize) is set to one so we can handle retrys more easily without worrying about idempotency in larger batches.
-* The `Metadata`'s Docker properties must be the same as our web function except for the `DockerTag`. This is needed for the image to be shared. This works around a known [SAM issue](https://github.com/aws/aws-sam-cli/issues/2466) vs using the `ImageConfig` property.
-* The jobs function `Timeout` must be lower than the `JobsQueue`'s `VisibilityTimeout` property. When the batch size is one, the queue's visibility is generally one second more.
+- The `Events` property uses the [SQS Type](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-sqs.html).
+- The [BatchSize](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-sqs.html#sam-function-sqs-batchsize) can be any number you like. Less means more Lambda concurrency, more means some jobs could take longer. The jobs function `Timeout` must be lower than the `JobsQueue`'s `VisibilityTimeout` property. When the batch size is one, the queue's visibility is generally one second more.
+- You must use `ReportBatchItemFailures` response types. Lambdakiq assumes we are [reporting batch item failures](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting). This is a new feature of SQS introduced in [November 2021](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html#services-sqs-batchfailurereporting).
+- The `Metadata`'s Docker properties must be the same as our web function except for the `DockerTag`. This is needed for the image to be shared. This works around a known [SAM issue](https://github.com/aws/aws-sam-cli/issues/2466) vs using the `ImageConfig` property.
 
 🎉 Deploy your application and have fun with ActiveJob on SQS & Lambda.
 
@@ -148,9 +178,9 @@ Most general Lambdakiq configuration options are exposed via the Rails standard
 config.lambdakiq
 ```
 
-* `max_retries=` - Retries for all jobs. Default is the Lambdakiq maximum of `12`.
-* `metrics_namespace=` - The CloudWatch Embedded Metrics namespace. Default is `Lambdakiq`.
-* `metrics_logger=` - Set to the Rails logger which is STDOUT via Lamby/Lambda.
+- `max_retries=` - Retries for all jobs. Default is the Lambdakiq maximum of `12`.
+- `metrics_namespace=` - The CloudWatch Embedded Metrics namespace. Default is `Lambdakiq`.
+- `metrics_logger=` - Set to the Rails logger which is STDOUT via Lamby/Lambda.
 
 ### ActiveJob Configs
 
@@ -162,13 +192,7 @@ class OrderProcessorJob < ApplicationJob
 end
 ```
 
-* `retry` - Overrides the default Lambdakiq `max_retries` for this one job.
-
-## Concurrency & Limits
-
-AWS SQS is highly scalable with [few limits](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-quotas.html). As your jobs in SQS increases so should your concurrent functions to process that work. However, as this article, ["Why isn't my Lambda function with an Amazon SQS event source scaling optimally?"](https://aws.amazon.com/premiumsupport/knowledge-center/lambda-sqs-scaling/) describes it is possible that errors will effect your concurrency.
-
-To help keep your queue and workers scalable, reduce the errors raised by your jobs. You an also reduce the retry count.
+- `retry` - Overrides the default Lambdakiq `max_retries` for this one job.
 
 ## Observability with CloudWatch
 
@@ -180,42 +204,43 @@ Metrics are published under the `Lambdakiq` namespace. This is configurable usin
 
 ### Metric Dimensions
 
-* `AppName` - This is the name of your Rails application. Ex: `MyApp`
-* `JobEvent` - Name of the ActiveSupport Notification. Ex: `*.active_job`.
-* `JobName` - The class name of the ActiveSupport job. Ex: `NotificationJob`
+- `AppName` - This is the name of your Rails application. Ex: `MyApp`
+- `JobEvent` - Name of the ActiveSupport Notification. Ex: `*.active_job`.
+- `JobName` - The class name of the ActiveSupport job. Ex: `NotificationJob`
 
 ### ActiveJob Event Names
+
 For reference, here are the `JobEvent` names published by ActiveSupport. A few of these are instrumented by Lambdakiq since we use custom retry logic like Sidekiq. These event/metrics are found in the Rails application CloudWatch logs because they publish/enqueue jobs.
 
-* `enqueue.active_job`
-* `enqueue_at.active_job`
+- `enqueue.active_job`
+- `enqueue_at.active_job`
 
 While these event/metrics can be found in the jobs function's log.
 
-* `perform_start.active_job`
-* `perform.active_job`
-* `enqueue_retry.active_job`
-* `retry_stopped.active_job`
+- `perform_start.active_job`
+- `perform.active_job`
+- `enqueue_retry.active_job`
+- `retry_stopped.active_job`
 
 ### Metric Properties
 
 These are the properties published with each metric. Remember, properties can not be used as metric data in charts but can be searched using [CloudWatch Logs Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AnalyzingLogData.html).
 
-* `JobId` - ActiveJob Unique ID. Ex: `9f3b6977-6afc-4769-aed6-bab1ad9a0df5`
-* `QueueName` - SQS Queue Name. Ex: `myapp-JobsQueue-14F18LG6XFUW5.fifo`
-* `MessageId` - SQS Message ID. Ex: `5653246d-dc5e-4c95-9583-b6b83ec78602`
-* `ExceptionName` - Class name of error raised. Present in perform and retry events.
-* `EnqueuedAt` - When ActiveJob enqueued the message. Ex: `2021-01-14T01:43:38Z`
-* `Executions` - The number of current executions. Counts from `1` and up.
-* `JobArg#{n}` - Enumerated serialized arguments.
+- `JobId` - ActiveJob Unique ID. Ex: `9f3b6977-6afc-4769-aed6-bab1ad9a0df5`
+- `QueueName` - SQS Queue Name. Ex: `myapp-JobsQueue-14F18LG6XFUW5.fifo`
+- `MessageId` - SQS Message ID. Ex: `5653246d-dc5e-4c95-9583-b6b83ec78602`
+- `ExceptionName` - Class name of error raised. Present in perform and retry events.
+- `EnqueuedAt` - When ActiveJob enqueued the message. Ex: `2021-01-14T01:43:38Z`
+- `Executions` - The number of current executions. Counts from `1` and up.
+- `JobArg#{n}` - Enumerated serialized arguments.
 
 ### Metric Data
 
 And finally, here are the metrics which each dimension can chart using [CloudWatch Metrics & Dashboards](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch_Dashboards.html).
 
-* `Duration` - Of the job event in milliseconds.
-* `Count` - Of the event.
-* `ExceptionCount` - Of the event. Useful with `ExceptionName`.
+- `Duration` - Of the job event in milliseconds.
+- `Count` - Of the event.
+- `ExceptionCount` - Of the event. Useful with `ExceptionName`.
 
 ### CloudWatch Dashboard Examples
 
@@ -223,15 +248,14 @@ Please share how you are using CloudWatch to monitor and/or alert on your Active
 
 💬 https://github.com/customink/lambdakiq/discussions/3
 
-
 ## Common Questions
 
 **Are Scheduled Jobs Supported?** - No. If you need a scheduled job please use the [SAM Schedule](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/sam-property-function-schedule.html) event source which invokes your function with an [Eventbridege AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html).
 
 **Are FIFO Queues Supported?** - Yes. When you create your [AWS::SQS::Queue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html) resources you can set the [FifoQueue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-sqs-queues.html#aws-sqs-queue-fifoqueue) property to `true`. Remember that both your jobs queue and the redrive queue must be the same. When using FIFO we:
 
-* Simulate `delay_seconds` for ActiveJob's wait by using visibility timeouts under the hood. We still cap it to non-FIFO's 15 minutes.
-* Set both the messages `message_group_id` and `message_deduplication_id` to the unique job id provided by ActiveJob.
+- Simulate `delay_seconds` for ActiveJob's wait by using visibility timeouts under the hood. We still cap it to non-FIFO's 15 minutes.
+- Set both the messages `message_group_id` and `message_deduplication_id` to the unique job id provided by ActiveJob.
 
 **Can I Use Multiple Queues?** - Yes. Nothing is stopping you from creating any number of queues and/or functions to process them. Your subclasses can use ActiveJob's `queue_as` method as needed. This is an easy way to handle job priorities too.
 

data/lib/lambdakiq/error.rb

@@ -2,16 +2,6 @@ module Lambdakiq
   class Error < StandardError
   end
 
-  class JobError < Error
-    attr_reader :original_exception, :job
-
-    def initialize(error)
-      @original_exception = error
-      super(error.message)
-      set_backtrace Rails.backtrace_cleaner.clean(error.backtrace)
-    end
-  end
-
   class FifoDelayError < Error
     def initialize(error)
       super

data/lib/lambdakiq/job.rb

@@ -9,9 +9,9 @@ module Lambdakiq
         records = Event.records(event)
         jobs = records.map { |record| new(record) }
         jobs.each(&:perform)
-        jwerror = jobs.detect{ |j| j.error }
-        return unless jwerror
-        raise JobError.new(jwerror.error)
+        failed_jobs = jobs.select { |j| j.error }
+        item_failures = failed_jobs.map { |j| { itemIdentifier: j.provider_job_id } }
+        { batchItemFailures: item_failures }
       end
 
     end
@@ -40,10 +40,14 @@ module Lambdakiq
       active_job.executions
     end
 
+    def provider_job_id
+      active_job.provider_job_id
+    end
+
     def perform
       if fifo_delay?
         fifo_delay
-        raise FifoDelayError, active_job.job_id
+        return
       end
       execute
     end
@@ -104,6 +108,7 @@ module Lambdakiq
     end
 
     def fifo_delay
+      @error = FifoDelayError.new(active_job.job_id)
       params = client_params.merge visibility_timeout: record.fifo_delay_visibility_timeout
       client.change_message_visibility(params)
     end

data/lib/lambdakiq/metrics.rb

@@ -16,6 +16,7 @@ module Lambdakiq
     end
 
     def log
+      return unless lambdakiq?
       logger.info JSON.dump(message)
     end
 
@@ -29,6 +30,16 @@ module Lambdakiq
       job.class.name
     end
 
+    def adapter_name
+      event.payload[:adapter].class.name
+    end
+
+    def lambdakiq?
+      adapter_name.include?('Lambdakiq') && 
+        job.respond_to?(:lambdakiq?) && 
+        job.lambdakiq?
+    end
+
     def logger
       Lambdakiq.config.metrics_logger
     end
@@ -51,6 +62,7 @@ module Lambdakiq
     end
 
     def instrument!
+      return unless lambdakiq?
       put_metric 'Duration', event.duration.to_i, 'Milliseconds'
       put_metric 'Count', 1, 'Count'
       put_metric 'ExceptionCount', 1, 'Count' if exception_name
@@ -59,7 +71,7 @@ module Lambdakiq
       set_property 'QueueName', job.queue_name
       set_property 'MessageId', job.provider_job_id if job.provider_job_id
       set_property 'ExceptionName', exception_name if exception_name
-      set_property 'EnqueuedAt', job.enqueued_at if job.enqueued_at
+      set_property 'EnqueuedAt', job.enqueued_at if job.respond_to?(:enqueued_at) && job.enqueued_at
       set_property 'Executions', job.executions if job.executions
       job.arguments.each_with_index do |argument, index|
         set_property "JobArg#{index+1}", argument
@@ -107,4 +119,3 @@ module Lambdakiq
 
   end
 end
-

data/lib/lambdakiq/version.rb

@@ -1,3 +1,3 @@
 module Lambdakiq
-  VERSION = '1.0.1'
+  VERSION = '2.0.0'
 end

data/lib/lambdakiq/worker.rb

@@ -16,6 +16,10 @@ module Lambdakiq
 
     end
 
+    def lambdakiq?
+      true
+    end
+
     def lambdakiq_retry
       lambdakiq_options_hash[:retry]
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lambdakiq
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0
 platform: ruby
 authors:
 - Ken Collins
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob