pallets 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 6bd6ebdb788c39b7d3cb434b60f94d927dacd47c
-  data.tar.gz: b463178c06885308a2192bbcf3d549c52391acbd
+SHA256:
+  metadata.gz: e275565357790942bf8c3d92e452921a5650122cf12679a399bafe67818ea75c
+  data.tar.gz: 3f8c9c584b71385fd679c93cdf208b924e45efea43562062ded98b7a83e05d86
 SHA512:
-  metadata.gz: 7e976084d28bba01327e0a8c41b8a948e27f6b3041d6a3a31665825d66b57225867ec410e9bf4f24afcf13241f41b855e1150601f37de37425fc72ee9582b5a4
-  data.tar.gz: c05e17653dac7716b9f52958b0bfb295517d614f5f97a00ffb91ad7100e83fbbf20f526f08f8315a1344829d01bd992c4920a7b0881aec4fea9b68eb0d259ff3
+  metadata.gz: 0e197b5b10ba2e569b8e7df245a607046831af3412b5417dc3084e986af27088240c61de963a9e60e6a1431921014c92a09b9f4dd0aec9003c6dd3ffe0ce84f4
+  data.tar.gz: cdfa840985dfcea75efea45f7dc5bc42e0aed4933cd8e02c996e46f4d8a58300c1feb47fe5fd5a6f46930b50e5b782525ed9c41434cb4cf21afcbe4ad8158c4e

data/.travis.yml

@@ -4,10 +4,9 @@ services:
   - redis-server
 cache: bundler
 rvm:
-  - 2.3.8
-  - 2.4.5
-  - 2.5.3
-  - 2.6.0
+  - 2.4.6
+  - 2.5.5
+  - 2.6.3
 before_install:
   # Bundler 2.0 needs a newer RubyGems
   - gem update --system

data/CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.0] - 2019-05-12
+### Added
+- wrap job execution with middleware (#38)
+- use `Middleware::JobLogger` for job logging (#39)
+- allow Appsignal instrumentation using `Middleware::AppsignalInstrumenter` (#40)
+
+### Removed
+- support for Ruby 2.3 (#41)
+
 ## [0.4.0] - 2019-04-07
 ### Added
 - give up workflow before it finishes by returning `false` in any of its tasks (#25)
@@ -47,6 +56,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## 0.1.0 - 2018-09-29
 - Pallets' inception <3
 
-[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/linkyndy/pallets/compare/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/linkyndy/pallets/compare/v0.4.0...v0.5.0
+[0.4.0]: https://github.com/linkyndy/pallets/compare/v0.3.0...v0.5.0
 [0.3.0]: https://github.com/linkyndy/pallets/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/linkyndy/pallets/compare/v0.1.0...v0.2.0

data/Gemfile

@@ -3,6 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'rake'
+gem 'appsignal'
 
 group :test do
   gem 'rspec'

data/examples/appsignal.rb

@@ -0,0 +1,32 @@
+require 'pallets'
+require 'pallets/middleware/appsignal_instrumenter'
+
+Appsignal.config = Appsignal::Config.new(
+  File.expand_path(File.dirname(__FILE__)),
+  "development"
+)
+Appsignal.start
+Appsignal.start_logger
+
+Pallets.configure do |c|
+  c.middleware << Pallets::Middleware::AppsignalInstrumenter
+end
+
+class Appsignaling < Pallets::Workflow
+  task 'Signaling'
+  task 'ReturningSignal' => 'Signaling'
+end
+
+class Signaling < Pallets::Task
+  def run
+    puts context['signal']
+  end
+end
+
+class ReturningSignal < Pallets::Task
+  def run
+    puts 'Ho!'
+  end
+end
+
+Appsignaling.new(signal: 'Hey').run

data/examples/config/appsignal.yml

@@ -0,0 +1,12 @@
+default: &defaults
+  push_api_key: "<%= ENV['APPSIGNAL_PUSH_API_KEY'] %>"
+  name: "Pallets"
+  debug: true
+
+development:
+  <<: *defaults
+  active: true
+
+production:
+  <<: *defaults
+  active: true

data/examples/config_savvy.rb

@@ -1,5 +1,12 @@
 require 'pallets'
 
+class AnnounceProcessing
+  def self.call(worker, job, context)
+    puts "Starting to process job..."
+    yield
+  end
+end
+
 Pallets.configure do |c|
   # Harness 4 Pallets workers per process
   c.concurrency = 4
@@ -23,6 +30,10 @@ Pallets.configure do |c|
   # Jobs will be retried up to 5 times upon failure. After that, they will be
   # given up. Retry times are exponential and happen after: 7, 22, 87, 262, ...
   c.max_failures = 5
+
+  # Job execution can be wrapped with middleware to provide custom logic.
+  # Anything that responds to `call` would do
+  c.middleware << AnnounceProcessing
 end
 
 class ConfigSavvy < Pallets::Workflow

data/lib/pallets.rb

@@ -9,6 +9,8 @@ require 'pallets/errors'
 require 'pallets/graph'
 require 'pallets/logger'
 require 'pallets/manager'
+require 'pallets/middleware/job_logger'
+require 'pallets/middleware/stack'
 require 'pallets/pool'
 require 'pallets/scheduler'
 require 'pallets/serializers/base'
@@ -50,15 +52,15 @@ module Pallets
     end
   end
 
+  def self.middleware
+    @middleware ||= configuration.middleware
+  end
+
   def self.logger
-    @logger ||= begin
-      logger = Pallets::Logger.new(STDOUT)
-      # TODO: Ruby 2.4 supports Logger initialization with the arguments below, so
-      #       we can drop this after we drop support for Ruby 2.3
-      logger.level = Pallets::Logger::INFO
-      logger.formatter = Pallets::Logger::Formatters::Pretty.new
-      logger
-    end
+    @logger ||= Pallets::Logger.new(STDOUT,
+      level: Pallets::Logger::INFO,
+      formatter: Pallets::Logger::Formatters::Pretty.new
+    )
   end
 
   def self.logger=(logger)

data/lib/pallets/configuration.rb

@@ -30,6 +30,15 @@ module Pallets
     # Serializer used for jobs
     attr_accessor :serializer
 
+    # Middleware used to wrap job execution with custom logic. Acts like a stack
+    # and accepts callable objects (lambdas, procs, objects that respond to call)
+    # that take three arguments: the worker handling the job, the job hash and
+    # the context
+    #
+    # A minimal example of a middleware is:
+    #   ->(worker, job, context, &b) { puts 'Hello World!'; b.call }
+    attr_reader :middleware
+
     def initialize
       @backend = :redis
       @backend_args = {}
@@ -39,10 +48,17 @@ module Pallets
       @job_timeout = 1_800 # 30 minutes
       @max_failures = 3
       @serializer = :json
+      @middleware = default_middleware
     end
 
     def pool_size
       @pool_size || @concurrency + 1
     end
+
+    def default_middleware
+      Middleware::Stack[
+        Middleware::JobLogger
+      ]
+    end
   end
 end

data/lib/pallets/middleware/appsignal_instrumenter.rb

@@ -0,0 +1,46 @@
+require 'appsignal'
+
+module Pallets
+  module Middleware
+    class AppsignalInstrumenter
+      extend Appsignal::Hooks::Helpers
+
+      def self.call(worker, job, context)
+        job_status = nil
+        transaction = Appsignal::Transaction.create(
+          SecureRandom.uuid,
+          Appsignal::Transaction::BACKGROUND_JOB,
+          Appsignal::Transaction::GenericRequest.new(queue_start: job['created_at'])
+        )
+
+        Appsignal.instrument('perform_job.pallets') do
+          begin
+            yield
+          rescue Exception => ex
+            job_status = :failed
+            transaction.set_error(ex)
+            raise
+          ensure
+            transaction.set_action_if_nil("#{job['task_class']}#run (#{job['workflow_class']})")
+            transaction.params = filtered_context(context)
+            formatted_metadata(job).each { |kv| transaction.set_metadata(*kv) }
+            transaction.set_http_or_background_queue_start
+            Appsignal::Transaction.complete_current!
+            Appsignal.increment_counter('pallets_job_count', 1, status: job_status || :successful)
+          end
+        end
+      end
+
+      def self.filtered_context(context)
+        Appsignal::Utils::HashSanitizer.sanitize(
+          context,
+          Appsignal.config[:filter_parameters]
+        )
+      end
+
+      def self.formatted_metadata(job)
+        job.map { |k, v| [k, truncate(string_or_inspect(v))] }
+      end
+    end
+  end
+end

data/lib/pallets/middleware/job_logger.rb

@@ -0,0 +1,26 @@
+module Pallets
+  module Middleware
+    class JobLogger
+      def self.call(worker, job, context)
+        Pallets.logger.info 'Started', extract_metadata(worker.id, job)
+        result = yield
+        Pallets.logger.info 'Done', extract_metadata(worker.id, job)
+        result
+      rescue => ex
+        Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(worker.id, job)
+        Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(worker.id, job) unless ex.backtrace.nil?
+        raise
+      end
+
+      def self.extract_metadata(wid, job)
+        {
+          wid:  wid,
+          wfid: job['wfid'],
+          jid:  job['jid'],
+          wf:   job['workflow_class'],
+          tsk:  job['task_class'],
+        }
+      end
+    end
+  end
+end

data/lib/pallets/middleware/stack.rb

@@ -0,0 +1,13 @@
+module Pallets
+  module Middleware
+    # Array-like class that acts like a stack and additionally provides the
+    # means to wrap an operation with callable objects
+    class Stack < Array
+      def invoke(*args, &block)
+        reverse.inject(block) do |memo, middleware|
+          lambda { middleware.call(*args, &memo) }
+        end.call
+      end
+    end
+  end
+end

data/lib/pallets/serializers/json.rb

@@ -4,12 +4,11 @@ module Pallets
   module Serializers
     class Json < Base
       def dump(data)
-        # TODO: Remove option after dropping support for Ruby 2.3
-        JSON.generate(data, quirks_mode: true)
+        JSON.generate(data)
       end
 
       def load(data)
-        JSON.parse(data, quirks_mode: true)
+        JSON.parse(data)
       end
     end
   end

data/lib/pallets/version.rb

@@ -1,3 +1,3 @@
 module Pallets
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/pallets/worker.rb

@@ -69,8 +69,6 @@ module Pallets
         return
       end
 
-      Pallets.logger.info "Started", extract_metadata(job_hash)
-
       context = Context[
         serializer.load_context(backend.get_context(job_hash['wfid']))
       ]
@@ -78,7 +76,9 @@ module Pallets
       task_class = Pallets::Util.constantize(job_hash["task_class"])
       task = task_class.new(context)
       begin
-        task_result = task.run
+        task_result = middleware.invoke(self, job_hash, context) do
+          task.run
+        end
       rescue => ex
         handle_job_error(ex, job, job_hash)
       else
@@ -91,8 +91,6 @@ module Pallets
     end
 
     def handle_job_error(ex, job, job_hash)
-      Pallets.logger.warn "#{ex.class.name}: #{ex.message}", extract_metadata(job_hash)
-      Pallets.logger.warn ex.backtrace.join("\n"), extract_metadata(job_hash) unless ex.backtrace.nil?
       failures = job_hash.fetch('failures', 0) + 1
       new_job = serializer.dump(job_hash.merge(
         'failures' => failures,
@@ -106,7 +104,6 @@ module Pallets
         backend.retry(new_job, job, retry_at)
       else
         backend.give_up(new_job, job)
-        Pallets.logger.info "Gave up after #{failures} failed attempts", extract_metadata(job_hash)
       end
     end
 
@@ -116,22 +113,10 @@ module Pallets
         'reason' => 'returned_false'
       ))
       backend.give_up(new_job, job)
-      Pallets.logger.info "Gave up after returning false", extract_metadata(job_hash)
     end
 
     def handle_job_success(context, job, job_hash)
       backend.save(job_hash['wfid'], job, serializer.dump_context(context.buffer))
-      Pallets.logger.info "Done", extract_metadata(job_hash)
-    end
-
-    def extract_metadata(job_hash)
-      {
-        wid:  id,
-        wfid: job_hash['wfid'],
-        jid: job_hash['jid'],
-        wf:   job_hash['workflow_class'],
-        tsk:  job_hash['task_class']
-      }
     end
 
     def backoff_in_seconds(count)
@@ -145,5 +130,9 @@ module Pallets
     def serializer
       @serializer ||= Pallets.serializer
     end
+
+    def middleware
+      @middleware ||= Pallets.middleware
+    end
   end
 end

data/pallets.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = ['pallets']
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.3'
+  spec.required_ruby_version = '>= 2.4'
 
   spec.add_dependency 'redis'
   spec.add_dependency 'msgpack'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pallets
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Andrei Horak
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-07 00:00:00.000000000 Z
+date: 2019-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -56,6 +56,8 @@ files:
 - README.md
 - Rakefile
 - bin/pallets
+- examples/appsignal.rb
+- examples/config/appsignal.yml
 - examples/config_savvy.rb
 - examples/do_groceries.rb
 - examples/hello_world.rb
@@ -75,6 +77,9 @@ files:
 - lib/pallets/graph.rb
 - lib/pallets/logger.rb
 - lib/pallets/manager.rb
+- lib/pallets/middleware/appsignal_instrumenter.rb
+- lib/pallets/middleware/job_logger.rb
+- lib/pallets/middleware/stack.rb
 - lib/pallets/pool.rb
 - lib/pallets/scheduler.rb
 - lib/pallets/serializers/base.rb
@@ -98,15 +103,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Toy workflow engine, written in Ruby