funktor 0.2.17 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7b7b575e4abcc04a0b771e3fb9afb912ee82015ed7aa6bcb17570fca0f392fa
-  data.tar.gz: 98d8caf556c78960b73a1ee40831d4165b01836831d65eb17014239450a5dce7
+  metadata.gz: 70f99a435a3ace385ee1dc3741517b03994a57fdd87e167299ef9407793cee18
+  data.tar.gz: b510965d08f62dab3891a33921da4d17e9bfcf2d5ee671dffea6ee3b2ef4e0d0
 SHA512:
-  metadata.gz: 46b2e852ec64c6f4f5ea23422ec987e95941583f51fe8d76a254df36093c543a4b250c4de5f35abd32df1f9fd8db96df6655b397248697c134a208b8bc60d6f9
-  data.tar.gz: a7231805102b4a7b05ce8eade0c35c4798779e6ca88171a294137bc20a334e32da47927abdb607d00f83d3585d7dec0d5706c2f6a6baab2e5cbf769b768e4532
+  metadata.gz: 5cabd0e3283726d5880fb2d29aeba2277a33fb5daac978843721aac9c6fbf1f94fcc3947755c9d35db42b98291a8c1465cbd03a2fa112dbdca8f44a421b69b0a
+  data.tar.gz: 2265f5459d090e444727338d7e5691dfcffd65c1b777456017b4a1b577ebae65fc52b07956fb78de949ac9c6e6aac4b174fa0e9991dab5427b1b661d859a3c7d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    funktor (0.2.17)
+    funktor (0.3.0)
       activesupport
       aws-sdk-sqs (~> 1.37)
       thor
@@ -81,4 +81,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.1.4
+   2.2.24

data/lib/funktor.rb

@@ -3,16 +3,21 @@ require 'funktor/aws/sqs/event'
 require 'funktor/aws/sqs/record'
 require 'funktor/counter'
 require 'funktor/job'
+require 'funktor/logger'
 require 'funktor/worker'
 require 'funktor/middleware_chain'
 require 'funktor/incoming_job_handler'
-require 'funktor/active_job_handler'
 
 require 'json'
 
 module Funktor
   class Error < StandardError; end
 
+  DEFAULT_OPTIONS = {
+    error_handlers: [],
+    log_level: Logger::DEBUG # Set a high log level during early, active development
+  }
+
   def self.configure_job_pusher
     yield self
   end
@@ -55,9 +60,41 @@ module Funktor
   def self.dump_json(object)
     JSON.generate(object)
   end
+
+  def self.options
+    @options ||= DEFAULT_OPTIONS.dup
+  end
+
+  def self.options=(opts)
+    @options = opts
+  end
+
+  # Register a proc to handle any error which occurs within the Funktor active job handler.
+  #
+  #   Funktor.error_handlers << proc {|error, context| ErrorsAsAService.notify(error, context) }
+  #
+  # The default error handler logs errors to STDOUT
+  def self.error_handlers
+    options[:error_handlers]
+  end
+
+  def self.logger
+    @logger ||= Funktor::Logger.new($stdout, level: options[:log_level])
+  end
+
+  def self.logger=(logger)
+    if logger.nil?
+      self.logger.level = Logger::FATAL
+      return self.logger
+    end
+
+    @logger = logger
+  end
 end
 
 # TODO - Should we require this by default or let people opt in?
-# Is it a code smell that we need to include it at the bottom, after
+# TODO - Is it a code smell that we need to include these at the bottom, after
 # the main Funktor module is defined?
 require 'funktor/middleware/metrics'
+require 'funktor/error_handler'
+require 'funktor/active_job_handler'

data/lib/funktor/active_job_handler.rb

@@ -2,6 +2,7 @@ require 'aws-sdk-sqs'
 
 module Funktor
   class ActiveJobHandler
+    include Funktor::ErrorHandler
 
     def initialize
       @failed_counter = Funktor::Counter.new('failed')
@@ -28,8 +29,7 @@ module Funktor
         @processed_counter.incr(job)
       # rescue Funktor::Job::InvalidJsonError # TODO Make this work
       rescue Exception => e
-        puts "Error during processing: #{$!}"
-        puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+        handle_error(e, job)
         @failed_counter.incr(job)
         attempt_retry_or_bail(job)
       end

data/lib/funktor/cli/bootstrap.rb

@@ -9,6 +9,10 @@ module Funktor
         :type => :string, :desc => "The bootstrap file to generate.",
         :default => "funktor_init.yml"
 
+      class_option :directory, :aliases => "-d",
+        :type => :string, :desc => "The directory in which to place the bootstrap file.",
+        :default => nil
+
       desc <<~DESC
         Description:
           Bootstrap a new funktor application by generating a funktor_init.yml file."
@@ -26,7 +30,11 @@ module Funktor
 
       private
       def funktor_file_target
-        File.join name, options[:file]
+        File.join funktor_directory_target, options[:file]
+      end
+
+      def funktor_directory_target
+        options[:directory] || name
       end
 
     end

data/lib/funktor/cli/init.rb

@@ -32,11 +32,11 @@ module Funktor
 
       def funktor_config_yml
         #template "funktor_config.yml", File.join("funktor_config.yml")
-        template File.join("config", "funktor.yml"), File.join("config", "funktor.yml")
-        template File.join("config", "ruby_layer.yml"), File.join("config", "ruby_layer.yml")
-        template File.join("config", "package.yml"), File.join("config", "package.yml")
-        template File.join("config", "environment.yml"), File.join("config", "environment.yml")
-        template File.join("config", "boot.rb"), File.join("config", "boot.rb")
+        template File.join("funktor_config", "funktor.yml"), File.join("funktor_config", "funktor.yml")
+        template File.join("funktor_config", "ruby_layer.yml"), File.join("funktor_config", "ruby_layer.yml")
+        template File.join("funktor_config", "package.yml"), File.join("funktor_config", "package.yml")
+        template File.join("funktor_config", "environment.yml"), File.join("funktor_config", "environment.yml")
+        template File.join("funktor_config", "boot.rb"), File.join("funktor_config", "boot.rb")
       end
 
       def package_json
@@ -56,33 +56,33 @@ module Funktor
       end
 
       def resources
-        template File.join("config", "resources", "incoming_job_queue.yml"), File.join("config", "resources", "incoming_job_queue.yml")
-        template File.join("config", "resources", "incoming_job_queue_user.yml"), File.join("config", "resources", "incoming_job_queue_user.yml")
+        template File.join("funktor_config", "resources", "incoming_job_queue.yml"), File.join("funktor_config", "resources", "incoming_job_queue.yml")
+        template File.join("funktor_config", "resources", "incoming_job_queue_user.yml"), File.join("funktor_config", "resources", "incoming_job_queue_user.yml")
         # TODO - Figure out how to make the dashboard aware of various queues...
-        template File.join("config", "resources", "cloudwatch_dashboard.yml"), File.join("config", "resources", "cloudwatch_dashboard.yml")
+        template File.join("funktor_config", "resources", "cloudwatch_dashboard.yml"), File.join("funktor_config", "resources", "cloudwatch_dashboard.yml")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
-          template File.join("config", "resources", "work_queue.yml"), File.join("config", "resources", "#{work_queue_name.underscore}_queue.yml")
+          template File.join("funktor_config", "resources", "work_queue.yml"), File.join("funktor_config", "resources", "#{work_queue_name.underscore}_queue.yml")
         end
       end
 
       def iam_permissions
-        template File.join("config", "iam_permissions", "ssm.yml"), File.join("config", "iam_permissions", "ssm.yml")
-        template File.join("config", "iam_permissions", "incoming_job_queue.yml"), File.join("config", "iam_permissions", "incoming_job_queue.yml")
+        template File.join("funktor_config", "iam_permissions", "ssm.yml"), File.join("funktor_config", "iam_permissions", "ssm.yml")
+        template File.join("funktor_config", "iam_permissions", "incoming_job_queue.yml"), File.join("funktor_config", "iam_permissions", "incoming_job_queue.yml")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
-          template File.join("config", "iam_permissions", "work_queue.yml"), File.join("config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
+          template File.join("funktor_config", "iam_permissions", "work_queue.yml"), File.join("funktor_config", "iam_permissions", "#{work_queue_name.underscore}_queue.yml")
         end
       end
 
       def function_definitions
-        template File.join("config", "function_definitions", "incoming_job_handler.yml"), File.join("config", "function_definitions", "incoming_job_handler.yml")
+        template File.join("funktor_config", "function_definitions", "incoming_job_handler.yml"), File.join("funktor_config", "function_definitions", "incoming_job_handler.yml")
         queues.each do |queue_details|
           @work_queue_name = queue_details.keys.first
           @work_queue_config = queue_details.values.first
-          template File.join("config", "function_definitions", "work_queue_handler.yml"), File.join("config", "function_definitions", "#{work_queue_name.underscore}_queue_handler.yml")
+          template File.join("funktor_config", "function_definitions", "work_queue_handler.yml"), File.join("funktor_config", "function_definitions", "#{work_queue_name.underscore}_queue_handler.yml")
         end
       end
 
@@ -112,15 +112,15 @@ module Funktor
       end
 
       def all_iam_permissions
-        Dir.glob(File.join('config', 'iam_permissions', '**.yml'))
+        Dir.glob(File.join('funktor_config', 'iam_permissions', '**.yml'))
       end
 
       def all_function_definitions
-        Dir.glob(File.join('config', 'function_definitions', '**.yml'))
+        Dir.glob(File.join('funktor_config', 'function_definitions', '**.yml'))
       end
 
       def all_resources
-        Dir.glob(File.join('config', 'resources', '**.yml'))
+        Dir.glob(File.join('funktor_config', 'resources', '**.yml'))
       end
 
       def funktor_config

data/lib/funktor/cli/templates/{config → funktor_config}/environment.yml

@@ -1,3 +1,4 @@
+BUNDLE_WITHOUT: development:test
 FUNKTOR_APP_NAME: <%= app_name %>
 FUNKTOR_INCOMING_JOB_QUEUE:
   Ref: IncomingJobQueue

data/lib/funktor/cli/templates/{config → funktor_config}/package.yml

@@ -3,7 +3,7 @@ individually: false
 include:
   - Gemfile
   - Gemfile.lock
-  - config/boot.rb
+  - funktor_config/boot.rb
   - app/**
 # Evertyting is excluded by default with serverless-ruby-layer, but you could use
 # the lines below to exlude files that are inside an include path.

data/lib/funktor/cli/templates/funktor_init.yml.tt

@@ -66,4 +66,4 @@ package:
     - Gemfile
     - Gemfile.lock
     - app/**
-    - config/**
+    - funktor_config/**

data/lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb

@@ -1,7 +1,4 @@
-# For this handler we don't need to know about your app, or any of the other gems,
-# so instead of doing `require_relative '../config/boog'` we just manually require
-# the one gem that we do need.
-require 'funktor'
+require_relative '../funktor_config/boot'
 
 $handler = Funktor::IncomingJobHandler.new
 

data/lib/funktor/cli/templates/lambda_event_handlers/work_queue_handler.rb

@@ -1,4 +1,4 @@
-require_relative '../config/boot'
+require_relative '../funktor_config/boot'
 
 $handler = Funktor::ActiveJobHandler.new
 

data/lib/funktor/cli/templates/serverless.yml

@@ -24,7 +24,7 @@ provider:
   name: aws
   runtime: <%= runtime %>
   lambdaHashingVersion: 20201221
-  environment: ${file(config/environment.yml)}
+  environment: ${file(funktor_config/environment.yml)}
   versionFunctions: false # Reduces the amount of storage used since all Lambdas together are limited to 75GB
   iamRoleStatements:
     <%- all_iam_permissions.each do |iam_permission| -%>
@@ -36,10 +36,10 @@ custom:
   # Our stage is based on what is passed in when running serverless
   # commands. Or fallsback to what we have set in the provider section.
   stage: ${self:provider.stage, 'dev'}
-  funktor: ${file(config/funktor.yml)}
-  rubyLayer: ${file(config/ruby_layer.yml)}
+  funktor: ${file(funktor_config/funktor.yml)}
+  rubyLayer: ${file(funktor_config/ruby_layer.yml)}
 
-package: ${file(config/package.yml)}
+package: ${file(funktor_config/package.yml)}
 
 functions:
   <%- all_function_definitions.each do |function_definition| -%>

data/lib/funktor/error_handler.rb

@@ -0,0 +1,25 @@
+module Funktor
+  module ErrorHandler
+    class Logger
+      def call(error, context)
+        Funktor.logger.warn(Funktor.dump_json(context)) if context
+        Funktor.logger.warn("#{error.class.name}: #{error.message}")
+        Funktor.logger.warn(error.backtrace.join("\n")) unless error.backtrace.nil?
+      end
+
+      Funktor.error_handlers << Funktor::ErrorHandler::Logger.new
+    end
+
+    def handle_error(error, context = {})
+      Funktor.error_handlers.each do |handler|
+        begin
+          handler.call(error, context)
+        rescue => new_error
+          Funktor.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+          Funktor.logger.error new_error
+          Funktor.logger.error new_error.backtrace.join("\n") unless new_error.backtrace.nil?
+        end
+      end
+    end
+  end
+end

data/lib/funktor/logger.rb

@@ -0,0 +1,5 @@
+require 'logger'
+module Funktor
+  class Logger < ::Logger
+  end
+end

data/lib/funktor/version.rb

@@ -1,3 +1,3 @@
 module Funktor
-  VERSION = "0.2.17"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: funktor
 version: !ruby/object:Gem::Version
-  version: 0.2.17
+  version: 0.3.1
 platform: ruby
 authors:
 - Jeremy Green
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-07-02 00:00:00.000000000 Z
+date: 2021-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-sqs
@@ -116,20 +116,20 @@ files:
 - lib/funktor/cli/init.rb
 - lib/funktor/cli/templates/Gemfile
 - lib/funktor/cli/templates/app/workers/hello_worker.rb
-- lib/funktor/cli/templates/config/boot.rb
-- lib/funktor/cli/templates/config/environment.yml
-- lib/funktor/cli/templates/config/function_definitions/incoming_job_handler.yml
-- lib/funktor/cli/templates/config/function_definitions/work_queue_handler.yml
-- lib/funktor/cli/templates/config/funktor.yml
-- lib/funktor/cli/templates/config/iam_permissions/incoming_job_queue.yml
-- lib/funktor/cli/templates/config/iam_permissions/ssm.yml
-- lib/funktor/cli/templates/config/iam_permissions/work_queue.yml
-- lib/funktor/cli/templates/config/package.yml
-- lib/funktor/cli/templates/config/resources/cloudwatch_dashboard.yml
-- lib/funktor/cli/templates/config/resources/incoming_job_queue.yml
-- lib/funktor/cli/templates/config/resources/incoming_job_queue_user.yml
-- lib/funktor/cli/templates/config/resources/work_queue.yml
-- lib/funktor/cli/templates/config/ruby_layer.yml
+- lib/funktor/cli/templates/funktor_config/boot.rb
+- lib/funktor/cli/templates/funktor_config/environment.yml
+- lib/funktor/cli/templates/funktor_config/function_definitions/incoming_job_handler.yml
+- lib/funktor/cli/templates/funktor_config/function_definitions/work_queue_handler.yml
+- lib/funktor/cli/templates/funktor_config/funktor.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/incoming_job_queue.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/ssm.yml
+- lib/funktor/cli/templates/funktor_config/iam_permissions/work_queue.yml
+- lib/funktor/cli/templates/funktor_config/package.yml
+- lib/funktor/cli/templates/funktor_config/resources/cloudwatch_dashboard.yml
+- lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue.yml
+- lib/funktor/cli/templates/funktor_config/resources/incoming_job_queue_user.yml
+- lib/funktor/cli/templates/funktor_config/resources/work_queue.yml
+- lib/funktor/cli/templates/funktor_config/ruby_layer.yml
 - lib/funktor/cli/templates/funktor_init.yml.tt
 - lib/funktor/cli/templates/gitignore
 - lib/funktor/cli/templates/lambda_event_handlers/incoming_job_handler.rb
@@ -137,9 +137,11 @@ files:
 - lib/funktor/cli/templates/package.json
 - lib/funktor/cli/templates/serverless.yml
 - lib/funktor/counter.rb
+- lib/funktor/error_handler.rb
 - lib/funktor/fake_job_queue.rb
 - lib/funktor/incoming_job_handler.rb
 - lib/funktor/job.rb
+- lib/funktor/logger.rb
 - lib/funktor/middleware/metrics.rb
 - lib/funktor/middleware_chain.rb
 - lib/funktor/testing.rb
@@ -166,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.24
 signing_key:
 specification_version: 4
 summary: Background processing in AWS Lambda.