glass_octopus 1.0.0

data/lib/glass_octopus/consumer.rb

@@ -0,0 +1,39 @@
+require "glass_octopus/unit_of_work"
+
+module GlassOctopus
+  # @api private
+  class Consumer
+    attr_reader :connection, :processor, :executor, :logger
+
+    def initialize(connection, processor, executor, logger)
+      @connection = connection
+      @processor  = processor
+      @executor   = executor
+      @logger     = logger
+    end
+
+    def run
+      connection.fetch_message do |message|
+        work = UnitOfWork.new(message, processor, logger)
+        submit(work)
+      end
+    end
+
+    def shutdown(timeout=10)
+      connection.close
+      executor.shutdown
+      logger.info("Waiting for workers to terminate...")
+      executor.wait_for_termination(timeout)
+    end
+
+    def submit(work)
+      if executor.post(work) { |work| work.perform }
+        logger.debug { "Accepted message: #{work.message.to_h}" }
+      else
+        logger.warn { "Rejected message: #{work.message.to_h}" }
+      end
+    rescue Concurrent::RejectedExecutionError
+      logger.warn { "Rejected message: #{work.message.to_h}" }
+    end
+  end
+end

data/lib/glass_octopus/context.rb

@@ -0,0 +1,30 @@
+require "forwardable"
+
+module GlassOctopus
+  # Message context. Wraps a Kafka message and adds some convenience methods.
+  #
+  # @!attribute [rw] logger
+  #   A logger object. Defaults to the application logger.
+  # @!attribute [r] message
+  #   A message read from Kafka.
+  #   @return [Message]
+  class Context
+    extend Forwardable
+    attr_reader :message
+    attr_accessor :logger
+
+    # @!method [](key)
+    #   Retrieves the +value+ object corresponding to the +key+ object.
+    #   @param key key to retrieve
+    # @!method []=(key, value)
+    #   Associates +value+ with +key+.
+    def_delegators :@data, :[], :[]=
+
+    # @api private
+    def initialize(message, logger)
+      @data    = {}
+      @message = message
+      @logger  = logger
+    end
+  end
+end

data/lib/glass_octopus/message.rb

@@ -0,0 +1,4 @@
+module GlassOctopus
+  # Represents a message from a Kafka topic.
+  Message = Struct.new(:topic, :partition, :offset, :key, :value)
+end

data/lib/glass_octopus/middleware.rb

@@ -0,0 +1,10 @@
+module GlassOctopus
+  module Middleware
+    autoload :ActiveRecord, "glass_octopus/middleware/active_record"
+    autoload :CommonLogger, "glass_octopus/middleware/common_logger"
+    autoload :JsonParser, "glass_octopus/middleware/json_parser"
+    autoload :Mongoid, "glass_octopus/middleware/mongoid"
+    autoload :NewRelic, "glass_octopus/middleware/new_relic"
+    autoload :Sentry, "glass_octopus/middleware/sentry"
+  end
+end

data/lib/glass_octopus/middleware/active_record.rb

@@ -0,0 +1,21 @@
+begin
+  require "active_record"
+rescue LoadError
+  raise "Can't find 'activerecord' gem. Please add it to your Gemfile or install it."
+end
+
+module GlassOctopus
+  module Middleware
+    class ActiveRecord
+      def initialize(app)
+        @app = app
+      end
+
+      def call(ctx)
+        @app.call(ctx)
+      ensure
+        ::ActiveRecord::Base.clear_active_connections!
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/common_logger.rb

@@ -0,0 +1,31 @@
+require "benchmark"
+
+module GlassOctopus
+  module Middleware
+    class CommonLogger
+      FORMAT = "Processed message. topic=%s, partition=%d, key=%s, runtime=%fms".freeze
+
+      def initialize(app, logger=nil, log_level=:info)
+        @app = app
+        @logger = logger
+        @log_level = log_level
+      end
+
+      def call(ctx)
+        log(ctx) { @app.call(ctx) }
+      end
+
+      private
+
+      def log(ctx)
+        logger = @logger || ctx.logger
+
+        runtime = Benchmark.realtime { yield }
+        runtime *= 1000 # Convert to milliseconds
+
+        logger.send(@log_level, format(FORMAT,
+          ctx.message.topic, ctx.message.partition, ctx.message.key, runtime))
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/json_parser.rb

@@ -0,0 +1,42 @@
+require "json"
+require "delegate"
+
+module GlassOctopus
+  module Middleware
+    class JsonParser
+      def initialize(app, options={})
+        @app = app
+        @klass = options.delete(:class)
+        @encoding = options.delete(:encoding)
+        @options = options
+      end
+
+      def call(ctx)
+        message = parse(ensure_encoding(ctx.message.value))
+        ctx = ContextWithJsonParsedMessage.new(ctx, message)
+        @app.call(ctx)
+      end
+
+      private
+
+      def parse(str)
+        hash = JSON.parse(str, { :create_additions => false }.merge(@options))
+        @klass ? @klass.new(hash) : hash
+      end
+
+      def ensure_encoding(value)
+        return value unless @encoding
+        value.encode(@encoding, invalid: :replace, undef: :replace, replace: '')
+      end
+
+      class ContextWithJsonParsedMessage < SimpleDelegator
+        attr_reader :params
+
+        def initialize(wrapped_ctx, params)
+          super(wrapped_ctx)
+          @params = params
+        end
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/mongoid.rb

@@ -0,0 +1,19 @@
+begin
+  require "mongoid"
+rescue LoadError
+  raise "Can't find 'mongoid' gem. Please add it to your Gemfile or install it."
+end
+
+module GlassOctopus
+  module Middleware
+    class Mongoid
+      def initialize(app)
+        @app = app
+      end
+
+      def call(ctx)
+        Mongoid.unit_of_work { @app.call(ctx) }
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/new_relic.rb

@@ -0,0 +1,32 @@
+begin
+  require 'new_relic/agent'
+rescue LoadError
+  raise "Can't find 'newrelic_rpm' gem. Please add it to your Gemfile or install it."
+end
+
+module GlassOctopus
+  module Middleware
+    class NewRelic
+      include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+      DEFAULT_OPTIONS = {
+        :name => "call",
+        :category => "OtherTransaction/GlassOctopus",
+      }.freeze
+
+      def initialize(app, klass, options={})
+        @app = app
+        @options = DEFAULT_OPTIONS.merge(class_name: klass.name).merge(options)
+      end
+
+      def call(ctx)
+        perform_action_with_newrelic_trace(@options) do
+          @app.call(ctx)
+        end
+      rescue Exception => ex
+        ::NewRelic::Agent.notice_error(ex, :custom_params => { :message => ctx.message.to_h })
+        raise
+      end
+    end
+  end
+end

data/lib/glass_octopus/middleware/sentry.rb

@@ -0,0 +1,33 @@
+begin
+  require "raven"
+rescue LoadError
+  raise "Can't find 'sentry-raven' gem. Please add it to your Gemfile or install it."
+end
+
+module GlassOctopus
+  module Middleware
+    class Sentry
+      def initialize(app)
+        @app = app
+      end
+
+      # Based on Raven::Rack integration
+      def call(ctx)
+        # clear context at the beginning of the processing to ensure a clean slate
+        Raven::Context.clear!
+        started_at = Time.now
+
+        begin
+          @app.call(ctx)
+        rescue Raven::Error
+          raise # Don't capture Raven errors
+        rescue Exception => ex
+          Raven.logger.debug("Collecting %p: %s" % [ ex.class, ex.message ])
+          Raven.capture_exception(ex, :extra => { :message => ctx.message.to_h },
+                                      :time_spent => Time.now - started_at)
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/glass_octopus/runner.rb

@@ -0,0 +1,64 @@
+require "singleton"
+
+module GlassOctopus
+  # A very simple runner that takes an app and handles graceful shutdown for
+  # SIGINT and SIGTERM.
+  #
+  # The {#run} method alters the state of the process globally and irreversibly
+  # by registering signal handlers. The Runner class is a singleton and can only
+  # be started once.
+  #
+  # Runner runs the application in the main thread. When a signal hits the
+  # process the control is transferred to the signal handler which will raise an
+  # Interrupt exception which kicks off the graceful shutdown. During shutdown
+  # no more messages are read because everything happens in the main thread.
+  #
+  # Runner does not provide any meaningful error handling. Errors are logged and
+  # then the process exits with status code 1.
+  class Runner
+    include Singleton
+
+    # Shortcut to {#run}.
+    # @return [void]
+    def self.run(app)
+      instance.run(app)
+    end
+
+    # Starts the application and blocks until the process gets a SIGTERM or
+    # SIGINT signal.
+    #
+    # @param app the application to run
+    # @return [void]
+    def run(app)
+      return if running?
+      running!
+
+      # To support JRuby Ctrl+C as MRI does.
+      # See: https://github.com/jruby/jruby/issues/1639
+      trap(:INT)  { Thread.main.raise Interrupt }
+      trap(:TERM) { Thread.main.raise Interrupt }
+
+      app.run
+    rescue Interrupt
+      app.logger.info("Shutting down...")
+      app.shutdown
+      app.logger.info("Bye.")
+    rescue => ex
+      app.logger.fatal("#{ex.class} - #{ex.message}:")
+      app.logger.fatal(ex.backtrace.join("\n")) if ex.backtrace
+      exit(1)
+    end
+
+    # Determines whether the application is running or not.
+    # @return [Boolean]
+    def running?
+      @running
+    end
+
+    private
+
+    def running!
+      @running = true
+    end
+  end
+end

data/lib/glass_octopus/unit_of_work.rb

@@ -0,0 +1,24 @@
+require "glass_octopus/context"
+
+module GlassOctopus
+  # Unit of work. Builds a context for a message and runs it through the
+  # middleware stack. It catches and logs all application level exceptions.
+  #
+  # @api private
+  class UnitOfWork
+    attr_reader :message, :processor, :logger
+
+    def initialize(message, processor, logger)
+      @message   = message
+      @processor = processor
+      @logger    = logger
+    end
+
+    def perform
+      processor.call(Context.new(message, logger))
+    rescue => ex
+      logger.logger.error("#{ex.class} - #{ex.message}:")
+      logger.logger.error(ex.backtrace.join("\n")) if ex.backtrace
+    end
+  end
+end

data/lib/glass_octopus/version.rb

@@ -0,0 +1,3 @@
+module GlassOctopus
+  VERSION = "1.0.0"
+end

metadata

@@ -0,0 +1,187 @@
+--- !ruby/object:Gem::Specification
+name: glass_octopus
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Tamás Michelberger
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-11-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: minitest-color
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.14'
+- !ruby/object:Gem::Dependency
+  name: guard-minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+description: |
+  GlassOctopus provides a minimal, modular and adaptable interface for developing
+  Kafka consumers in Ruby. In its philosophy it is very close to Rack.
+email:
+- tomi@secretsaucepartners.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env"
+- ".gitignore"
+- ".yardopts"
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/guard
+- bin/rake
+- docker-compose.yml
+- example/advanced.rb
+- example/basic.rb
+- example/ruby_kafka.rb
+- glass_octopus.gemspec
+- lib/glass-octopus.rb
+- lib/glass_octopus.rb
+- lib/glass_octopus/application.rb
+- lib/glass_octopus/bounded_executor.rb
+- lib/glass_octopus/builder.rb
+- lib/glass_octopus/configuration.rb
+- lib/glass_octopus/connection/options_invalid.rb
+- lib/glass_octopus/connection/poseidon_adapter.rb
+- lib/glass_octopus/connection/ruby_kafka_adapter.rb
+- lib/glass_octopus/consumer.rb
+- lib/glass_octopus/context.rb
+- lib/glass_octopus/message.rb
+- lib/glass_octopus/middleware.rb
+- lib/glass_octopus/middleware/active_record.rb
+- lib/glass_octopus/middleware/common_logger.rb
+- lib/glass_octopus/middleware/json_parser.rb
+- lib/glass_octopus/middleware/mongoid.rb
+- lib/glass_octopus/middleware/new_relic.rb
+- lib/glass_octopus/middleware/sentry.rb
+- lib/glass_octopus/runner.rb
+- lib/glass_octopus/unit_of_work.rb
+- lib/glass_octopus/version.rb
+homepage: https://github.com/sspinc/glass-octopus
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.2
+signing_key: 
+specification_version: 4
+summary: A Kafka consumer framework. Like Rack but for Kafka.
+test_files: []