deimos-kafka 1.0.0.pre.beta15

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+  task(default: :spec)
+rescue LoadError # rubocop:disable Lint/HandleExceptions
+  # no rspec available
+end
+
+import('./lib/tasks/phobos.rake')

data/bin/deimos

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'deimos'

data/deimos-kafka.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'deimos/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'deimos-kafka'
+  spec.version       = Deimos::VERSION
+  spec.authors       = ['Daniel Orner']
+  spec.email         = ['daniel.orner@wishabi.com']
+  spec.summary       = 'Kafka libraries for Ruby.'
+  spec.homepage      = ''
+  spec.license       = 'Apache 2.0'
+
+  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency('avro-patches', '~> 0.3')
+  spec.add_runtime_dependency('avro_turf', '~> 0.8')
+  spec.add_runtime_dependency('phobos', '~> 1.8')
+  spec.add_runtime_dependency('ruby-kafka', '~> 0.7')
+
+  spec.add_development_dependency('activerecord', '~> 5.2')
+  spec.add_development_dependency('activerecord-import')
+  spec.add_development_dependency('bundler', '~> 1')
+  spec.add_development_dependency('ddtrace', '~> 0.11')
+  spec.add_development_dependency('dogstatsd-ruby', '~> 4.2')
+  spec.add_development_dependency('guard', '~> 2')
+  spec.add_development_dependency('guard-rspec', '~> 4')
+  spec.add_development_dependency('guard-rubocop', '~> 1')
+  spec.add_development_dependency('mysql2', '~> 0.5')
+  spec.add_development_dependency('pg', '~> 1.1')
+  spec.add_development_dependency('rake', '~> 10')
+  spec.add_development_dependency('rspec', '~> 3')
+  spec.add_development_dependency('rspec_junit_formatter', '~>0.3')
+  spec.add_development_dependency('rubocop', '~> 0.72')
+  spec.add_development_dependency('rubocop-rspec', '~> 1.27')
+  spec.add_development_dependency('sqlite3', '~> 1.3')
+end

data/docker-compose.yml

@@ -0,0 +1,71 @@
+version: '3.6'
+services:
+  mysql:
+    image: mysql:5.7
+    expose:
+      - 3306
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_DATABASE=test
+      - TZ=America/Toronto
+
+  postgres:
+    image: postgres:11.1
+    expose:
+      - 5432
+    environment:
+      POSTGRES_PASSWORD: root
+
+  test:
+    volumes:
+      - .:/var/app
+    depends_on:
+      - kafka-broker
+      - mysql
+      - postgres
+    build: .
+    environment:
+      - "DEFAULT_TIMEOUT=${DEFAULT_TIMEOUT}"
+      - MYSQL_HOST=mysql
+      - PG_HOST=postgres
+      - SCHEMA_REGISTRY=http://schema-registry:8081
+      - KAFKA_SEED_BROKER=kafka-broker:9092
+    command: dockerize -wait tcp://mysql:3306 -wait tcp://postgres:5432 -timeout 1m rspec
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+     - 2181:2181
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry
+    hostname: schema-registry
+    depends_on:
+      - zookeeper
+      - kafka-broker
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+
+  kafka-broker:
+    image: confluentinc/cp-enterprise-kafka
+    hostname: kafka-broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-broker:9092'
+      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
+      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+

data/docs/DATABASE_BACKEND.md

@@ -0,0 +1,147 @@
+# Database Backend Design
+
+Kafka is a messaging protocol, while databases are transactional and relational. 
+Marrying the two (e.g. by using Kafka to publish changes to a database table)
+is not a simple task. This document describes the problem and the current
+implementation. There will be references to microservices architecture as that
+informs some of the discussion.
+
+## A Pure Solution
+
+The purest solution is to use Kafka as the "source of truth" by first publishing
+all messages synchronously to Kafka, and then using a consumer to read these 
+messages back into the database. If there are any errors sending to Kafka,
+the thread will crash and no data would be written. If there are errors
+reading the data back into the database, the data remains in Kafka and can
+be re-read at any time.
+
+There are several real-world problems with this pure solution:
+
+1. The solution assumes that creating a consumer is a simple task, which is 
+   definitely not the case. Depending on how many topics are being produced to,
+   a separate consumer thread per topic (which is how Phobos works) is overkill.
+   The other option is to introduce an entirely new consumer service to handle
+   consuming the topics we've already produced, which is even more overkill.
+2.  For CRUD interfaces or any other UI that saves data to the database, we do 
+    not want to use an asynchronous method to ensure that data is published to 
+    Kafka before saving, and then being able to serve that data back to the user
+    in a single API call, which is a common use case. 
+    This could involve a large amount of added complexity and may force the user 
+    to wait unnecessarily.
+3.  We want to make use of database transactions - i.e. if an error happens 
+    saving one record, the others should roll back. Once a message is written to 
+    Kafka, it can't be "rolled back" easily. Kafka transactions do exist but 
+    they are not widely supported in Kafka clients, and we still would be 
+    faced with the fact that one transaction (Kafka or database) could finish 
+    and then the process could be killed before the other one could be finished.
+4.  We want to make use of auto-increment database IDs - we can't do this if we 
+    write to Kafka first.
+5.  Kafka is an external dependency. If either the DB **or** Kafka goes down, 
+    our app becomes unusable.
+
+Using tools like Kafka Connect and Debezium are not ideal because:
+
+1. They are tied very closely to the internal relational schema, which is not 
+   ideal, especially for legacy systems. It makes it nearly impossible to make 
+   internal changes.
+2. They are separate services and connectors must be created for each 
+   microservice separately, which is a large overhead.
+
+## Database Backend Solution
+
+We will be using the database itself as the source of our Kafka messages. 
+We will first write our messages to a database table and then asynchronously 
+send those messages to Kafka. This solves our problems:
+
+1.  The database is the (interim) source of truth. The Kafka message log is 
+    essentially the changelog, which we can tail and send out. If our producing 
+    thread errors out, a new one will simply pick up where it left off. 
+    This ensures eventual consistency.
+2.  Because we are only using the database in the main application thread, we do 
+    not need to wait for Kafka production to continue and can return immediately.
+3.  Because we are only saving to the database, we can use transactions normally 
+    - if a transaction fails, it will roll back along with any Kafka messages we 
+    intended to send.
+4.  Records are saved normally and messages are created after that, all as part 
+    of the transaction, so we can use database IDs as usual.
+5.  We remove Kafka entirely as a dependency for normal work - the Kafka sending 
+    piece is a separate thread.
+    
+The one downside to this is a slight delay (generally less than 1 second) 
+between the message being written to the database and sent to Kafka - in most 
+cases this is an acceptable limitation.
+
+### The Implementation
+
+The database backend consists of three tables:
+
+* `kafka_messages` - this keeps track of the messages that were "published",
+  including the payload, topic, key and partition key. These messages
+  are *raw data* - all processing, including Avro encoding, must happen
+  upstream before they are inserted.
+* `kafka_topic_info` - this table is essentially a lock table used to ensure
+  that only one producer thread is ever "working" on a topic at a time.
+
+The backend code structure is such that when a producer calls `publish_list`,
+it delegates that logic to the configured backend. A backend of `kafka`
+or `kafka_async` will use existing Phobos logic. A backend of `db` will use
+the database backend instead.
+
+### "Publishing" A Message
+
+When `publish_list` is called when the database backend is configured, 
+Deimos will instead save the message to the `kafka_messages` table.
+
+### Sending Messages to Kafka
+
+The database executor is started by calling `Deimos.start_db_backend!`
+with a specified number of threads. These threads will continually scan the
+two messages tables and send the messages to Kafka. 
+
+The algorithm for sending the messages makes use of the `kafka_topic_info` table as a lock table. There is also an `error` boolean column which is used to track when a topic has errored out. When this happens, the topic is marked as errored and will not be picked up for the next minute, after which it will be treated as any other topic. The full algorithm is as follows:
+
+* Create a UUID for the thread - this is created once on thread start.
+* Find all unique topics in the `kafka_messages` table.
+* For each topic:
+  * Create an entry in `kafka_topic_info` for this topic if it doesn't exist.
+  * Run the following query: 
+  
+      ```sql
+            UPDATE kafka_topic_info 
+            SET locked_by=#{uuid}, locked_at=NOW(), error=0
+            WHERE (locked_by IS NULL AND error=0) OR locked_at < #{1.minute.ago} 
+            LIMIT 1
+      ```
+     * If the lock was unsuccessful, move on to the next topic in the list
+  * Find the first 1000 messages in `kafka_messages` for that topic, ordered by ID (insertion order)
+  * Send the messages synchronously to Kafka, with all brokers acking the message.
+  * Delete the records from the DB
+  * Update the `locked_at` timestamp in `kafka_topic_info` to `NOW()` to ensure liveness in case a particular batch took longer than expected to send.
+  * If the current batch is 1000 messages, repeat with the next batch of
+    messages until it returns less than 1000
+  * When all batches are sent:
+      * Unlock the topic by updating the `kafka_topic_info` for this topic, setting `locked_by=NULL, locked_at=NULL, error=0, retries=0`
+      * Move on to the next topic
+   * If there are errors sending a batch:
+     * Update the `kafka_topic_info` for this topic to have `locked_by=NULL, locked_at=NULL, error=1, retries=retries+1` - this will effectively keep it
+       locked for the next minute
+     * Move on to the next topic.
+* When all topics are done, or if there are no topics, sleep for 0.5 seconds and begin again.
+
+### Class / Method Design
+
+The algorithm is split up into the following classes:
+
+* Backends::Db - this is the class that saves the message to the database.
+* KafkaMessage: This is an ActiveRecord class that handle saving the messages to the database and querying them.
+* KafkaTopicInfo: This is an ActiveRecord class that handles locking, unlocking and heartbeating.
+* Utils::SignalHandler: This is the equivalent of Phobos's Runner class and
+  handles the KILL, INT and TERM signals to gracefully shut down the threads.
+* Utils::Executor is the equivalent of Phobos's Executor class and handles
+  the thread pool of producer threads.
+* Utils::DbProducer is the producer thread itself which implements most of the
+  algorithm listed above.
+
+### Caveats
+
+There is one disadvantage of this pattern, which is that it is possible for events to be sent multiple times if the thread which sent the messages dies before being able to delete it from the database. In general this is an acceptable effect, since Kafka only guarantees at-least-once delivery in any case.

data/docs/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,34 @@
+# Pull Request Template
+
+## Description
+
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+Fixes # (issue)
+
+## Type of change
+
+Please delete options that are not relevant.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+## How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Test A
+- [ ] Test B
+
+## Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

data/lib/deimos.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'avro-patches'
+require 'avro_turf'
+require 'phobos'
+require 'deimos/version'
+require 'deimos/avro_data_encoder'
+require 'deimos/avro_data_decoder'
+require 'deimos/producer'
+require 'deimos/active_record_producer'
+require 'deimos/active_record_consumer'
+require 'deimos/consumer'
+require 'deimos/configuration'
+require 'deimos/instrumentation'
+require 'deimos/utils/lag_reporter'
+
+require 'deimos/publish_backend'
+require 'deimos/backends/kafka'
+require 'deimos/backends/kafka_async'
+
+require 'deimos/monkey_patches/ruby_kafka_heartbeat'
+require 'deimos/monkey_patches/schema_store'
+require 'deimos/monkey_patches/phobos_producer'
+require 'deimos/monkey_patches/phobos_cli'
+
+require 'deimos/railtie' if defined?(Rails)
+if defined?(ActiveRecord)
+  require 'deimos/kafka_source'
+  require 'deimos/kafka_topic_info'
+  require 'deimos/backends/db'
+  require 'deimos/utils/signal_handler.rb'
+  require 'deimos/utils/executor.rb'
+  require 'deimos/utils/db_producer.rb'
+end
+require 'deimos/utils/inline_consumer'
+require 'yaml'
+require 'erb'
+
+# Parent module.
+module Deimos
+  class << self
+    attr_accessor :config
+
+    # Configure Deimos.
+    def configure
+      first_time_config = self.config.nil?
+      self.config ||= Configuration.new
+      old_config = self.config.dup
+      yield(config)
+
+      # Don't re-configure Phobos every time
+      if first_time_config || config.phobos_config_changed?(old_config)
+
+        file = config.phobos_config_file
+        phobos_config = YAML.load(ERB.new(File.read(File.expand_path(file))).result)
+
+        configure_kafka_for_phobos(phobos_config)
+        configure_loggers(phobos_config)
+
+        Phobos.configure(phobos_config)
+      end
+
+      validate_db_backend if self.config.publish_backend == :db
+    end
+
+    # Ensure everything is set up correctly for the DB backend.
+    def validate_db_backend
+      begin
+        require 'activerecord-import'
+      rescue LoadError
+        raise 'Cannot set publish_backend to :db without activerecord-import! Please add it to your Gemfile.'
+      end
+      if Phobos.config.producer_hash[:required_acks] != :all
+        raise 'Cannot set publish_backend to :db unless required_acks is set to ":all" in phobos.yml!'
+      end
+    end
+
+    # Start the DB producers to send Kafka messages.
+    # @param thread_count [Integer] the number of threads to start.
+    def start_db_backend!(thread_count: 1)
+      return if self.config.publish_backend != :db ||
+                thread_count.nil? ||
+                thread_count.zero?
+
+      producers = (1..thread_count).map do
+        Deimos::Utils::DbProducer.new(self.config.logger)
+      end
+      executor = Deimos::Utils::Executor.new(producers,
+                                             self.config.logger)
+      signal_handler = Deimos::Utils::SignalHandler.new(executor)
+      run_db_backend_in_thread(signal_handler)
+    end
+
+    # @param signal_handler [Deimos::Utils::SignalHandler]
+    def run_db_backend_in_thread(signal_handler)
+      Thread.new { signal_handler.run! }
+    end
+
+    # @param phobos_config [Hash]
+    def configure_kafka_for_phobos(phobos_config)
+      if config.ssl_enabled
+        %w(ssl_ca_cert ssl_client_cert ssl_client_cert_key).each do |key|
+          next if config.send(key).blank?
+
+          phobos_config['kafka'][key] = ssl_var_contents(config.send(key))
+        end
+      end
+      phobos_config['kafka']['seed_brokers'] = config.seed_broker if config.seed_broker
+    end
+
+    # @param phobos_config [Hash]
+    def configure_loggers(phobos_config)
+      phobos_config['custom_logger'] = config.phobos_logger
+      phobos_config['custom_kafka_logger'] = config.kafka_logger
+    end
+
+    # @param filename [String] a file to read, or the contents of the SSL var
+    # @return [String] the contents of the file
+    def ssl_var_contents(filename)
+      File.exist?(filename) ? File.read(filename) : filename
+    end
+  end
+end
+
+at_exit do
+  begin
+    Deimos::Backends::KafkaAsync.producer.async_producer_shutdown
+    Deimos::Backends::KafkaAsync.producer.kafka_client&.close
+  rescue StandardError => e
+    Deimos.config.logger.error(
+      "Error closing async producer on shutdown: #{e.message} #{e.backtrace.join("\n")}"
+    )
+  end
+end