funktor 0.2.10 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16ad166a922c31ac7de1008d48fe34824203b0d325e7faae2f7ede140fc0f836
-  data.tar.gz: fafbd5ca0858ecb87c2c9e87c8efa52accc9262f922d6f3f1d4b311b3c8c470b
+  metadata.gz: a1b4182ab20dd1edaab9d00d00f454d25a1e0333502b2c20fca839902692f6ba
+  data.tar.gz: dc5d7f97ddda5d20292db6520d7023c185cf5f117a548b8c40d3dcaf96608061
 SHA512:
-  metadata.gz: b40e4deee38ed584b93c863150d2d6c45dfc3aea1ac3ed0d7aa4d249c5661b527f252cf1b700545f54e80f98bf6158345c01b218f175eef117ce31626c5f5b21
-  data.tar.gz: 240839bae2964df499664ec8c21a44d3c8c4cf46d3d6df5cb798af95ba3b454deee6a41d5b4b0a3d71fc8c04e36bdfdb0d787f6168e71ef1cb516370e1c0d369
+  metadata.gz: 0ae29280dd7ed160ac7f228a5ca0fdde92634e8bcd63179ffcacd49107b680a2d72f3067878d277c7146d72ccc0d2321575e7bc46acf9925a717ab32973772f6
+  data.tar.gz: 562723cc07a29e5a3036610423bbc54ad4ed00ede8ec47462319750924802b3c8bd37c9a0ea3235b455db9864a08dd628ef592fe6c72f10727ef7675468527a4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.10)
+    funktor (0.2.15)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor

data/lib/funktor.rb

@@ -1,6 +1,7 @@
 require "funktor/version"
 require 'funktor/aws/sqs/event'
 require 'funktor/aws/sqs/record'
+require 'funktor/counter'
 require 'funktor/job'
 require 'funktor/worker'
 require 'funktor/middleware_chain'

data/lib/funktor/active_job_handler.rb

@@ -3,6 +3,11 @@ require 'aws-sdk-sqs'
 module Funktor
   class ActiveJobHandler
 
+    def initialize
+      @failed_counter = Funktor::Counter.new('failed')
+      @processed_counter = Funktor::Counter.new('processed')
+    end
+
     def call(event:, context:)
       event = Funktor::Aws::Sqs::Event.new(event)
       puts "event.jobs.count = #{event.jobs.count}"
@@ -20,10 +25,12 @@ module Funktor
         Funktor.active_job_handler_middleware.invoke(job) do
           job.execute
         end
+        @processed_counter.incr(job)
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
         puts "Error during processing: #{$!}"
         puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+        @failed_counter.incr(job)
         attempt_retry_or_bail(job)
       end
     end
@@ -43,8 +50,7 @@ module Funktor
       puts job.to_json
       sqs_client.send_message({
         queue_url: job.retry_queue_url,
-        message_body: job.to_json,
-        delay_seconds: job.delay
+        message_body: job.to_json
       })
     end
 

data/lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml

@@ -321,6 +321,29 @@ Resources:
                         "title": "Job Duration by Worker"
                     }
                 },
+
+                <% current_y += 6 %>
+                { <% "Processed/Failed Jobs By Worker" %>
+                    "height": 6,
+                    "width": 24,
+                    "y": <%= current_y %>,
+                    "x": 0,
+                    "type": "metric",
+                    "properties": {
+                        "metrics": [
+                            <%- app_worker_names.each do |worker_name| -%>
+                            [ "<%= app_name %>", "processed", "WorkerClassName", "<%= worker_name %>" ],
+                            [ ".", "failed", ".", "." ]<%= worker_name == app_worker_names.last ? "" : "," %>
+                            <%- end -%>
+                        ],
+                        "view": "timeSeries",
+                        "stacked": false,
+                        "region": "us-east-1",
+                        "title": "Process/Failed Jobs By Worker",
+                        "period": 60,
+                        "stat": "Sum"
+                  }
+                },
                 <% current_y += 6 %>
 
                 { <% "Delayed Jobs" %>
@@ -551,5 +574,8 @@ Resources:
                     }
                 }
 
+
+                
+
             ]
         }

data/lib/funktor/counter.rb

@@ -0,0 +1,43 @@
+module Funktor
+  class Counter
+    attr_accessor :dimension
+
+    def initialize(dimension)
+      @dimension = dimension
+    end
+
+    def incr(job)
+      put_metric_to_stdout(job)
+    end
+
+    def put_metric_to_stdout(job)
+      puts Funktor.dump_json(metric_hash(job))
+    end
+
+    def metric_hash(job)
+      {
+        "_aws": {
+          "Timestamp": Time.now.strftime('%s%3N').to_i,
+          "CloudWatchMetrics": [
+            {
+              "Namespace": ENV['FUNKTOR_APP_NAME'],
+              "Dimensions": [["WorkerClassName", "Queue"]],
+              "Metrics": [ # CPU, Memory, Duration, etc...
+                           {
+                             "Name": dimension,
+                             "Unit": "Count"
+                           }
+              ]
+            }
+          ]
+        },
+        "WorkerClassName": job.worker_class_name,
+        "Queue": job.queue,
+        "#{dimension}": 1
+        #"count": value,
+        #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"
+      }
+      #data[key] = value
+    end
+  end
+end

data/lib/funktor/job.rb

@@ -35,7 +35,11 @@ module Funktor
     end
 
     def delay
-      job_data["delay"]
+      # TODO - In Funktor Pro we need to override this method (or do something else) so that
+      # we can schedule jobs farther in the future than 15 minutes. We do this here in case a
+      # retry sequence goes too long.
+      jdelay = job_data["delay"]
+      return jdelay < 900 ? jdelay : 900
     end
 
     def delay=(delay)

data/lib/funktor/middleware/metrics.rb

@@ -20,7 +20,7 @@ module Funktor
             "CloudWatchMetrics": [
               {
                 "Namespace": ENV['FUNKTOR_APP_NAME'],
-                "Dimensions": [["WorkerClassName"]],
+                "Dimensions": [["WorkerClassName", "Queue"]],
                 "Metrics": [ # CPU, Memory, Duration, etc...
                              {
                                "Name": "Duration",
@@ -31,6 +31,7 @@ module Funktor
             ]
           },
           "WorkerClassName": job.worker_class_name,
+          "Queue": job.queue,
           "Duration": time_diff_in_seconds * 1_000
           #"count": value,
           #"requestId": "989ffbf8-9ace-4817-a57c-e4dd734019ee"

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.10"
+  VERSION = "0.2.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.15
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-06-30 00:00:00.000000000 Z
+date: 2021-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -136,6 +136,7 @@ files:
 - lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
+- lib/funktor/counter.rb
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb