phobos 1.0.0

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "phobos"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+config_path = ENV['CONFIG_PATH'] || (File.exist?('config/phobos.yml') ? 'config/phobos.yml' : 'config/phobos.yml.example')
+Phobos.configure(config_path)
+
+require "irb"
+IRB.start

data/bin/phobos

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'phobos'
+require 'phobos/cli'
+
+STDOUT.sync = true
+Phobos::CLI::Commands.start(ARGV)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/circle.yml

@@ -0,0 +1,27 @@
+machine:
+  pre:
+    - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0
+  services:
+    - docker
+  environment:
+    LOG_LEVEL: DEBUG
+    CI: true
+    DEFAULT_TIMEOUT: 20
+  ruby:
+    version: 2.3.1
+
+dependencies:
+  pre:
+    - docker -v
+    - docker pull ches/kafka:0.9.0.1
+    - docker pull jplock/zookeeper:3.4.6
+    - gem install bundler -v 1.9.5
+    - bundle install
+
+test:
+  override:
+    - docker run -d -p 2003:2181 --name zookeeper jplock/zookeeper:3.4.6; sleep 5
+    - docker run -d -p 9092:9092 --name kafka -e KAFKA_BROKER_ID=0 -e KAFKA_ADVERTISED_HOST_NAME=localhost -e KAFKA_ADVERTISED_PORT=9092 -e ZOOKEEPER_CONNECTION_STRING=zookeeper:2181 --link zookeeper:zookeeper ches/kafka:0.9.0.1; sleep 5
+    - bundle exec rspec -r rspec_junit_formatter --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/unit.xml
+  post:
+    - cp log/*.log $CIRCLE_ARTIFACTS/ || true

data/config/phobos.yml.example

@@ -0,0 +1,78 @@
+logger:
+  file: log/phobos.log
+  level: info
+
+kafka:
+  # identifier for this application
+  client_id: phobos
+  # timeout setting for connecting to brokers
+  connect_timeout:
+  # timeout setting for socket connections
+  socket_timeout:
+  # PEM encoded CA cert to use with an SSL connection (string)
+  ssl_ca_cert:
+  # PEM encoded client cert to use with an SSL connection (string)
+  # Must be used in combination with ssl_client_cert_key
+  ssl_client_cert:
+  # PEM encoded client cert key to use with an SSL connection (string)
+  # Must be used in combination with ssl_client_cert
+  ssl_client_cert_key:
+  # list of brokers used to initialize the client ("port:protocol")
+  seed_brokers:
+    - localhost:9092
+
+producer:
+  # number of seconds a broker can wait for replicas to acknowledge
+  # a write before responding with a timeout
+  ack_timeout: 5
+  # number of replicas that must acknowledge a write, or `:all`
+  # if all in-sync replicas must acknowledge
+  required_acks: :all
+  # number of retries that should be attempted before giving up sending
+  # messages to the cluster. Does not include the original attempt
+  max_retries: 2
+  # number of seconds to wait between retries
+  retry_backoff: 1
+  # number of messages allowed in the buffer before new writes will
+  # raise {BufferOverflow} exceptions
+  max_buffer_size: 1000
+  # maximum size of the buffer in bytes. Attempting to produce messages
+  # when the buffer reaches this size will result in {BufferOverflow} being raised
+  max_buffer_bytesize: 10000000
+  # name of the compression codec to use, or nil if no compression should be performed.
+  # Valid codecs: `:snappy` and `:gzip`
+  compression_codec:
+  # number of messages that needs to be in a message set before it should be compressed.
+  # Note that message sets are per-partition rather than per-topic or per-producer
+  compression_threshold: 1
+
+consumer:
+  # number of seconds after which, if a client hasn't contacted the Kafka cluster,
+  # it will be kicked out of the group
+  session_timeout: 30
+  # interval between offset commits, in seconds
+  offset_commit_interval: 10
+  # number of messages that can be processed before their offsets are committed.
+  # If zero, offset commits are not triggered by message processing
+  offset_commit_threshold: 0
+  # interval between heartbeats; must be less than the session window
+  heartbeat_interval: 10
+
+backoff:
+  min_ms: 1000
+  max_ms: 60000
+
+listeners:
+  - handler: Phobos::EchoHandler
+    topic: test
+    # id of the group that the consumer should join
+    group_id: test-1
+    # Once the consumer group has checkpointed its progress in the topic's partitions,
+    # the consumers will always start from the checkpointed offsets, regardless of config
+    # As such, this setting only applies when the consumer initially starts consuming from a topic
+    start_from_beginning: true
+    # maximum amount of data fetched from a single partition at a time
+    max_bytes_per_partition: 524288 # 512 KB
+    # Number of threads created for this listener, each thread will behave as an independent consumer.
+    # They don't share any state
+    max_concurrency: 1

data/examples/handler_saving_events_database.rb

@@ -0,0 +1,49 @@
+#
+# This example assumes that you want to save all events in your database for
+# recovery purposes. The consumer will process the message and perform other
+# operations, this implementation assumes a generic way to save the events.
+#
+# Setup your database connection using `phobos_boot.rb`. Remember to Setup
+# a hook to disconnect, e.g: `at_exit { Database.disconnect! }`
+#
+class HandlerSavingEventsDatabase
+  include Phobos::Handler
+  include Phobos::Producer
+
+  def self.around_consume(payload, metadata)
+    #
+    # Let's assume `::from_message` will initialize our object with `payload`
+    #
+    event = Model::Event.from_message(payload)
+
+    #
+    # If event already exists in the database, skip this message
+    #
+    return if event.exists?
+
+    Model::Event.transaction do
+      #
+      # Executes `#consume` method
+      #
+      new_values = yield
+
+      #
+      # `#consume` method can return additional data (up to your code)
+      #
+      event.update_with_new_attributes(new_values)
+
+      #
+      # Let's assume the event is just initialized and now is the time to save it
+      #
+      event.save!
+    end
+  end
+
+  def consume(payload, metadata)
+    #
+    # Process the event, it might index it to elasticsearch or notify other
+    # system, you should process your message inside this method.
+    #
+    { new_vale: payload.length % 3 }
+  end
+end

data/examples/handler_using_async_producer.rb

@@ -0,0 +1,15 @@
+#
+# This example assumes you want to process the event and publish another
+# one to kafka. A new event is always published thus we want to use the async producer
+# to better use our resources and to speed up the process
+#
+class HandlerUsingAsyncProducer
+  include Phobos::Handler
+  include Phobos::Producer
+
+  PUBLISH_TO 'another-topic'
+
+  def consume(payload, metadata)
+    producer.async_publish(PUBLISH_TO, "#{payload}-#{rand}")
+  end
+end

data/examples/publishing_messages_without_consumer.rb.rb

@@ -0,0 +1,72 @@
+require "bundler/setup"
+require "phobos"
+
+TOPIC = 'test-partitions'
+
+Phobos.configure('config/phobos.yml')
+
+class MyProducer
+  include Phobos::Producer
+end
+
+#
+# Trapping signals to properly stop this generator
+#
+@stop = false
+%i( INT TERM QUIT ).each do |signal|
+  Signal.trap(signal) do
+    puts "Stopping"
+    @stop = true
+  end
+end
+
+Thread.new do
+  begin
+    total = 1
+
+    loop do
+      break if @stop
+      key = SecureRandom.uuid
+      payload = Time.now.utc.to_json
+
+      begin
+        # Producer will use phobos configuration to create a kafka client and
+        # a producer and it will bind both to the current thread, so it's safe
+        # to call class methods here
+        #
+        MyProducer
+          .producer
+          .async_publish(TOPIC, payload, key)
+
+        puts "produced #{key}, total: #{total}"
+
+      # Since this is a simplistic code we are going to generate more messages than
+      # the producer can write to Kafka, so eventually we'll get some buffer overflows
+      #
+      rescue Kafka::BufferOverflow => e
+        puts "| waiting"
+        sleep(1)
+        retry
+      end
+
+      total += 1
+    end
+  ensure
+    #
+    # Before we stop we must shutdown the async producer to ensure that all messages
+    # are delivered
+    #
+    MyProducer
+      .producer
+      .async_producer_shutdown
+
+    #
+    # Since no client was configured (we can do this with `MyProducer.producer.configure_kafka_client`)
+    # we must get the auto generated one and close it properly
+    #
+    MyProducer
+      .producer
+      .kafka_client
+      .close
+  end
+end.join

data/lib/phobos.rb

@@ -0,0 +1,62 @@
+require 'yaml'
+require 'securerandom'
+
+require 'concurrent'
+require 'kafka'
+require 'hashie'
+require 'logging'
+require 'exponential_backoff'
+require 'active_support/notifications'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/hash/keys'
+
+require 'phobos/version'
+require 'phobos/instrumentation'
+require 'phobos/errors'
+require 'phobos/listener'
+require 'phobos/producer'
+require 'phobos/handler'
+require 'phobos/echo_handler'
+require 'phobos/executor'
+
+Thread.abort_on_exception = true
+
+module Phobos
+  class << self
+    attr_reader :config, :logger
+    attr_accessor :silence_log
+
+    def configure(yml_path)
+      ENV['RAILS_ENV'] = ENV['RACK_ENV'] ||= 'development'
+      @config = Hashie::Mash.new(YAML.load_file(File.expand_path(yml_path)))
+      @config.class.send(:define_method, :producer_hash) { Phobos.config.producer&.to_hash&.symbolize_keys }
+      @config.class.send(:define_method, :consumer_hash) { Phobos.config.consumer&.to_hash&.symbolize_keys }
+      configure_logger
+      logger.info { Hash(message: 'Phobos configured', env: ENV['RACK_ENV']) }
+    end
+
+    def create_kafka_client
+      Kafka.new(config.kafka.to_hash.symbolize_keys)
+    end
+
+    def create_exponential_backoff
+      min = Phobos.config.backoff.min_ms / 1000.0
+      max = Phobos.config.backoff.max_ms / 1000.0
+      ExponentialBackoff.new(min, max).tap { |backoff| backoff.randomize_factor = rand }
+    end
+
+    def configure_logger
+      date_pattern = '%Y-%m-%dT%H:%M:%S:%L%zZ'
+      FileUtils.mkdir_p(File.dirname(config.logger.file))
+
+      Logging.backtrace true
+      Logging.logger.root.level = silence_log ? :fatal : config.logger.level
+
+      @logger = Logging.logger[self]
+      @logger.appenders = [
+        Logging.appenders.stdout(layout: Logging.layouts.pattern(date_pattern: date_pattern)),
+        Logging.appenders.file(config.logger.file, layout: Logging.layouts.json(date_pattern: date_pattern))
+      ]
+    end
+  end
+end

data/lib/phobos/cli.rb

@@ -0,0 +1,61 @@
+require 'thor'
+require 'phobos/cli/start'
+
+module Phobos
+  module CLI
+
+    def self.logger
+      @logger ||= Logging.logger[self].tap do |l|
+        l.appenders = [Logging.appenders.stdout]
+      end
+    end
+
+    class Commands < Thor
+      include Thor::Actions
+
+      map '-v' => :version
+      map '--version' => :version
+
+      desc 'version', 'Outputs the version number. Can be used with: phobos -v or phobos --version'
+      def version
+        puts Phobos::VERSION
+      end
+
+      desc 'init', 'Initialize your project with Phobos'
+      def init
+        copy_file 'config/phobos.yml.example', 'config/phobos.yml'
+        create_file 'phobos_boot.rb' do
+          <<~EXAMPLE
+            # Use this file to load your code
+            puts <<~ART
+              ______ _           _
+              | ___ \\\\ |         | |
+              | |_/ / |__   ___ | |__   ___  ___
+              |  __/| '_ \\\\ / _ \\\\| '_ \\\\ / _ \\\\/ __|
+              | |   | | | | (_) | |_) | (_) \\\\__ \\\\
+              \\\\_|   |_| |_|\\\\___/|_.__/ \\\\___/|___/
+            ART
+            puts "\nphobos_boot.rb - find this file at \#{File.expand_path(__FILE__)}\n\n"
+          EXAMPLE
+        end
+      end
+
+      desc 'start', 'Starts Phobos'
+      option :config,
+             aliases: ['-c'],
+             default: 'config/phobos.yml',
+             banner: 'Configuration file'
+      option :boot,
+             aliases: ['-b'],
+             banner: 'File path to load application specific code',
+             default: 'phobos_boot.rb'
+      def start
+        Phobos::CLI::Start.new(options).execute
+      end
+
+      def self.source_root
+        File.expand_path(File.join(File.dirname(__FILE__), '../..'))
+      end
+    end
+  end
+end

data/lib/phobos/cli/runner.rb

@@ -0,0 +1,48 @@
+module Phobos
+  module CLI
+    class Runner
+
+      SIGNALS = %i( INT TERM QUIT ).freeze
+
+      def initialize
+        @signal_queue = []
+        @reader, @writer = IO.pipe
+        @executor = Phobos::Executor.new
+      end
+
+      def run!
+        setup_signals
+        executor.start
+
+        loop do
+          case signal_queue.pop
+          when *SIGNALS
+            executor.stop
+            break
+          else
+            ready = IO.select([reader, writer])
+
+            # drain the self-pipe so it won't be returned again next time
+            reader.read_nonblock(1) if ready[0].include?(reader)
+          end
+        end
+      end
+
+      private
+
+      attr_reader :reader, :writer, :signal_queue, :executor
+
+      def setup_signals
+        SIGNALS.each do |signal|
+          Signal.trap(signal) { unblock(signal) }
+        end
+      end
+
+      def unblock(signal)
+        writer.write_nonblock('.')
+        signal_queue << signal
+      end
+
+    end
+  end
+end