phobos_db_checkpoint 0.5.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 780d29aed6d459e239c93d0bd1982964d9ecaaf8
-  data.tar.gz: b5e02880a25c59329ce422566e8d23acb59538c4
+  metadata.gz: 4b29e206cb7135f08b91717795503156869f4b7f
+  data.tar.gz: e5e38ea02d946d15df776e139de72943e85cbefc
 SHA512:
-  metadata.gz: 042e4635cd5939215f3afbc3468c07a26efbe7c3998a9c5484d21595f2a12ccdf978b44aa16970f55b4e1b11bb9a90da86b755eac67f16f617571dbfed731818
-  data.tar.gz: 7b24abd6b44be10026d371339f8b1dade902bb07dcec2ea99299661c8f5e3aa3c3c7c83b4fa215259aef5c6fb5ce1b0532e1a19446c633ff0169611e000b52c1
+  metadata.gz: a01c09788822ac5ecf0f11d412777b9a29274645ad46805ebb972da9e6469afec76402ea364d03bd61011ccab1d35912d4986dfa83550133eee757adc88c3125
+  data.tar.gz: f76968bd632cd8a0a5f98a39458d9d07af98a0cbba1c6596b9a015c7fb37e8821ac407cba2d6ec8e70b8f42f0f31c27d2095376f71a472772b4203b5bf05507c

data/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.0.0 (2017-02-24)
+
+- [feature] Introduce failures and failure handling in Handler. Add failures to the events API
+
 ## 0.5.0 (2016-12-28)
 
 - [feature] Add another instrumentation to wrap the entire around consume

data/README.md

@@ -14,6 +14,7 @@ Phobos DB Checkpoint is a plugin to [Phobos](https://github.com/klarna/phobos) a
 1. [Usage](#usage)
   1. [Setup](#setup)
   1. [Handler](#handler)
+    1. [Failures](#failures)
   1. [Accessing the events](#accessing-the-events)
   1. [Events API](#events-api)
   1. [Instrumentation](#instrumentation)
@@ -119,6 +120,66 @@ If your handler returns anything different than an __ack__ it won't be saved to
 
 Note that the `PhobosDBCheckpoint::Handler` will automatically skip already handled events (i.e. duplicate Kafka messages).
 
+#### <a name="failures"></a> Failures
+
+If your handler fails during the process of consuming the event, the event will be processed again acknowledged or skipped. The default behavior of `Phobos` is to back off but keep retrying the same event forever, in order to guarantee messages are processed in the correct order. However, this blocking process could go on indefinitely, so in order to help you deal with this PhobosDBCheckpoint can (on an opt-in basis) mark them as permanently failed after a configurable number of attempts.
+
+This configuration is set in Phobos configuration:
+
+```yml
+db_checkpoint:
+  max_retries: 3
+```
+
+The retry decision is driven by inspecting the retry counter in the Phobos metadata, and if not meeting the retry criteria it will result in creating a `Failure` record and then skipping the event. You can easily retry these events later by simply invoking `retry!` on them.
+
+Optionally, by overriding the `retry_consume?` method you can take control over the conditions that apply for retrying consumption. Whenever these are not met, a failing event will be moved out of the queue and become a Failure.
+
+The control is based on `payload` and `exception`:
+
+```ruby
+class MyHandler
+  include PhobosDBCheckpoint::Handler
+
+  def retry_consume?(event, event_metadata, exception)
+    event_metadata[:retry_count] <= MyApp.config.max_retries
+  end
+end
+```
+
+##### Failure details
+
+Since PhobosDBCheckpoint does not know about the internals of your payload, for setting certain fields it is necessary to yield control back to the application.
+In case you need to customize your failures, these are the methods you should implement in your handler:
+
+```ruby
+class MyHandler
+  include PhobosDBCheckpoint::Handler
+    def entity_id(payload)
+      # Extract event id...
+      payload['my_payload']['my_event_id']
+    end
+
+    def entity_time(payload)
+      # Extract event time...
+      payload['my_payload']['my_event_time']
+    end
+
+    def event_type(payload)
+      # Extract event type...
+      payload['my_payload']['my_event_type']
+    end
+
+    def event_version(payload)
+      # Extract event version...
+      payload['my_payload']['my_event_version']
+    end
+  end
+end
+```
+
+This is completely optional, and if a method is not implemented, the corresponding value will simply be set to null.
+
 ### <a name="accessing-the-events">Accessing the events</a>
 
 `PhobosDBCheckpoint::Event` is a plain `ActiveRecord::Base` model, feel free to play with it.

data/lib/phobos_db_checkpoint.rb

@@ -3,10 +3,14 @@ require 'digest'
 require 'active_record'
 require 'phobos'
 
+require 'phobos_db_checkpoint/event_helper'
 require 'phobos_db_checkpoint/version'
+require 'phobos_db_checkpoint/errors'
 require 'phobos_db_checkpoint/event_actions'
 require 'phobos_db_checkpoint/event'
+require 'phobos_db_checkpoint/failure'
 require 'phobos_db_checkpoint/handler'
+require 'phobos_db_checkpoint/actions/retry_failure'
 
 module PhobosDBCheckpoint
   DEFAULT_DB_DIR = 'db'.freeze

data/lib/phobos_db_checkpoint/actions/retry_failure.rb

@@ -0,0 +1,29 @@
+module PhobosDBCheckpoint
+  class RetryFailure
+    def initialize(failure)
+      @failure = failure
+    end
+
+    def perform
+      result = retry_failure!
+      @failure.destroy
+      result
+    end
+
+    private
+
+    def retry_failure!
+      handler
+        .consume(
+          @failure.payload,
+          @failure.metadata.merge(retry_count: 0)
+        )
+    end
+
+    def handler
+      @failure
+        .configured_handler
+        .new
+    end
+  end
+end

data/lib/phobos_db_checkpoint/errors.rb

@@ -0,0 +1,8 @@
+module PhobosDBCheckpoint
+  class PhobosDBCheckpointError < StandardError; end
+  class ListenerNotFoundError < PhobosDBCheckpointError
+    def initialize(group_id)
+      super("Phobos Listener not found for group id '#{group_id}'")
+    end
+  end
+end

data/lib/phobos_db_checkpoint/event.rb

@@ -1,5 +1,6 @@
 module PhobosDBCheckpoint
   class Event < ActiveRecord::Base
+    include PhobosDBCheckpoint::EventHelper
     after_initialize :assign_checksum
 
     def exists?
@@ -14,14 +15,6 @@ module PhobosDBCheckpoint
       save!
     end
 
-    def configured_handler
-      Phobos
-        .config
-        .listeners
-        .find { |listener| listener.group_id == self.group_id }
-        &.handler
-    end
-
     private
 
     def assign_checksum

data/lib/phobos_db_checkpoint/event_helper.rb

@@ -0,0 +1,31 @@
+module PhobosDBCheckpoint
+  module EventHelper
+    def configured_listener
+      listener = Phobos
+        .config
+        .listeners
+        .find { |l| l.group_id == self.group_id }
+
+      raise(ListenerNotFoundError, self.group_id) unless listener
+
+      listener
+    end
+
+    def configured_handler
+      configured_listener
+        .handler
+        .constantize
+    end
+
+    def method_missing(m, *args, &block)
+      rex = m.to_s.match /^fetch_(.+)/
+
+      if rex
+        handler = configured_handler.new
+        return handler.send(rex[1], payload) if handler.respond_to?(rex[1])
+      else
+        super
+      end
+    end
+  end
+end

data/lib/phobos_db_checkpoint/events_api.rb

@@ -41,16 +41,23 @@ module PhobosDBCheckpoint
     post "/#{VERSION}/events/:id/retry" do
       content_type :json
       event = PhobosDBCheckpoint::Event.find(params['id'])
-      handler_name = event.configured_handler
+      metadata = {
+        listener_id: 'events_api/retry',
+        group_id: event.group_id,
+        topic: event.topic,
+        retry_count: 0
+      }
 
-      unless handler_name
-        status 422
-        return { error: true, message: 'no handler configured for this event' }.to_json
-      end
-
-      handler_class = handler_name.constantize
-      metadata = { listener_id: 'events_api/retry', group_id: event.group_id, topic: event.topic, retry_count: 0 }
-      event_action = handler_class.new.consume(event.payload, metadata)
+      event_action =
+        begin
+          event
+            .configured_handler
+            .new
+            .consume(event.payload, metadata)
+        rescue ListenerNotFoundError => e
+          status 422
+          return { error: true, message: e.message }.to_json
+        end
 
       { acknowledged: event_action.is_a?(PhobosDBCheckpoint::Ack) }.to_json
     end
@@ -72,5 +79,41 @@ module PhobosDBCheckpoint
         .offset(offset)
         .to_json
     end
+
+    get "/#{VERSION}/failures" do
+      content_type :json
+
+      limit = (params['limit'] || 20).to_i
+      offset = (params['offset'] || 0).to_i
+
+      query = PhobosDBCheckpoint::Failure
+      query = query.where(topic: params['topic']) if params['topic']
+      query = query.where(group_id: params['group_id']) if params['group_id']
+      query = query.where(entity_id: params['entity_id']) if params['entity_id']
+      query = query.where(event_type: params['event_type']) if params['event_type']
+
+      query
+        .order(event_time: :desc)
+        .limit(limit)
+        .offset(offset)
+        .to_json
+    end
+
+    post "/#{VERSION}/failures/:id/retry" do
+      content_type :json
+      failure = PhobosDBCheckpoint::Failure.find(params['id'])
+
+      failure_action =
+        begin
+          PhobosDBCheckpoint::RetryFailure
+            .new(failure)
+            .perform
+        rescue => e
+          status 422
+          return { error: true, message: e.message }.to_json
+        end
+
+      { acknowledged: failure_action.is_a?(PhobosDBCheckpoint::Ack) }.to_json
+    end
   end
 end

data/lib/phobos_db_checkpoint/failure.rb

@@ -0,0 +1,40 @@
+module PhobosDBCheckpoint
+  class Failure < ActiveRecord::Base
+    include PhobosDBCheckpoint::EventHelper
+
+    def self.record(event:, event_metadata:, exception: nil)
+      return if exists?(event_metadata[:checksum])
+
+      create do |record|
+        record.topic           = event_metadata[:topic]
+        record.group_id        = event_metadata[:group_id]
+        record.entity_id       = event.fetch_entity_id
+        record.event_time      = event.fetch_event_time
+        record.event_type      = event.fetch_event_type
+        record.event_version   = event.fetch_event_version
+        record.checksum        = event_metadata[:checksum]
+        record.payload         = event.payload
+        record.metadata        = event_metadata
+        record.error_class     = exception&.class&.name
+        record.error_message   = exception&.message
+        record.error_backtrace = exception&.backtrace
+      end
+    end
+
+    def self.exists?(checksum)
+      where(checksum: checksum).exists?
+    end
+
+    def payload
+      attributes['payload'].deep_symbolize_keys
+    end
+
+    def metadata
+      attributes['metadata'].deep_symbolize_keys
+    end
+
+    def group_id
+      attributes['group_id'] || metadata[:group_id]
+    end
+  end
+end

data/lib/phobos_db_checkpoint/handler.rb

@@ -14,6 +14,11 @@ module PhobosDBCheckpoint
       include Phobos::Instrumentation
       include Phobos::Handler::ClassMethods
 
+      def retry_consume?(event, event_metadata, exception)
+        return true unless Phobos.config&.db_checkpoint&.max_retries
+        event_metadata[:retry_count] < Phobos.config&.db_checkpoint&.max_retries
+      end
+
       def around_consume(payload, metadata)
         event = PhobosDBCheckpoint::Event.new(
           topic: metadata[:topic],
@@ -31,7 +36,15 @@ module PhobosDBCheckpoint
           end
 
           event_action = instrument('db_checkpoint.event_action', event_metadata) do
-            yield
+            begin
+              yield
+            rescue => e
+              if retry_consume?(event, event_metadata, e)
+                raise e
+              else
+                Failure.record(event: event, event_metadata: event_metadata, exception: e)
+              end
+            end
           end
 
           case event_action

data/lib/phobos_db_checkpoint/version.rb

@@ -1,3 +1,3 @@
 module PhobosDBCheckpoint
-  VERSION = '0.5.0'
+  VERSION = '1.0.0'
 end

data/templates/migrate/phobos_02_create_failures.rb.erb

@@ -0,0 +1,23 @@
+class Phobos02CreateFailures < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def up
+    create_table :failures do |t|
+      t.timestamp :created_at,     index: false
+      t.string    :topic,          index: true
+      t.string    :group_id,       index: true
+      t.string    :entity_id,      index: true
+      t.timestamp :event_time,     index: true
+      t.string    :event_type,     index: true
+      t.string    :event_version,  index: true
+      t.string    :checksum,       index: true
+      t.json      :payload,        index: false
+      t.json      :metadata,       index: false
+      t.string    :error_class,    index: false
+      t.string    :error_message,  index: false
+      t.json      :error_backtrace
+    end
+  end
+
+  def down
+    drop_table :failures
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: phobos_db_checkpoint
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Túlio Ornelas
@@ -13,7 +13,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-28 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -251,10 +251,14 @@ files:
 - bin/setup
 - circle.yml
 - lib/phobos_db_checkpoint.rb
+- lib/phobos_db_checkpoint/actions/retry_failure.rb
 - lib/phobos_db_checkpoint/cli.rb
+- lib/phobos_db_checkpoint/errors.rb
 - lib/phobos_db_checkpoint/event.rb
 - lib/phobos_db_checkpoint/event_actions.rb
+- lib/phobos_db_checkpoint/event_helper.rb
 - lib/phobos_db_checkpoint/events_api.rb
+- lib/phobos_db_checkpoint/failure.rb
 - lib/phobos_db_checkpoint/handler.rb
 - lib/phobos_db_checkpoint/middleware/database.rb
 - lib/phobos_db_checkpoint/middleware/logger.rb
@@ -264,6 +268,7 @@ files:
 - templates/config.ru
 - templates/database.yml.example
 - templates/migrate/phobos_01_create_events.rb.erb
+- templates/migrate/phobos_02_create_failures.rb.erb
 - templates/new_migration.rb.erb
 - templates/phobos_boot.rb
 homepage: https://github.com/klarna/phobos_db_checkpoint
@@ -287,7 +292,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Phobos DB Checkpoint is a plugin to Phobos and is meant as a drop in replacement