flipp-ruby-kafka-temp-fork 0.0.1

data/Dockerfile

@@ -0,0 +1,21 @@
+FROM 421990735784.dkr.ecr.us-east-1.amazonaws.com/flipp/ruby-base:2.5-stretch
+
+RUN apt-get install -yq libpq-dev net-tools mysql-client wait-for-it
+ENV DOCKERIZE_VERSION v0.6.1
+RUN wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \
+    && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+
+WORKDIR /opt/flipp_ruby_kafka
+COPY flipp-ruby-kafka.gemspec /opt/flipp_ruby_kafka/flipp-ruby-kafka.gemspec
+COPY lib/flipp_ruby_kafka/version.rb /opt/flipp_ruby_kafka/lib/flipp_ruby_kafka/version.rb
+COPY Gemfile /opt/flipp_ruby_kafka/Gemfile
+COPY Gemfile.lock /opt/flipp_ruby_kafka/Gemfile.lock
+
+RUN bundle install
+
+ADD . .
+
+ENTRYPOINT ["bundle", "exec"]
+
+CMD ["bundle", "exec", "rspec"]

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://flipplib.jfrog.io/flipplib/api/gems/flipp-gems/'
+
+# Specify your gem's dependencies in boilerplate.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,207 @@
+PATH
+  remote: .
+  specs:
+    flipp-ruby-kafka-temp-fork (0.0.1)
+      ddtrace (>= 0.11)
+      deimos-temp-fork (>= 0.0.2)
+      flipp-ruby-dogstatsd (~> 1.0)
+      railties (>= 3)
+
+GEM
+  remote: https://flipplib.jfrog.io/flipplib/api/gems/flipp-gems/
+  specs:
+    actionpack (6.1.4)
+      actionview (= 6.1.4)
+      activesupport (= 6.1.4)
+      rack (~> 2.0, >= 2.0.9)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actionview (6.1.4)
+      activesupport (= 6.1.4)
+      builder (~> 3.1)
+      erubi (~> 1.4)
+      rails-dom-testing (~> 2.0)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activesupport (6.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    avro (1.9.2)
+      multi_json
+    avro_turf (0.11.0)
+      avro (>= 1.7.7, < 1.10)
+      excon (~> 0.45)
+    builder (3.2.4)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.9)
+    concurrent-ruby-ext (1.1.9)
+      concurrent-ruby (= 1.1.9)
+    crass (1.0.6)
+    ddtrace (0.52.0)
+      ffi (~> 1.0)
+      msgpack
+    deimos-temp-fork (0.0.2)
+      avro_turf (~> 0.11)
+      fig_tree (~> 0.0.2)
+      phobos_temp_fork
+      sigurd (~> 0.0.1)
+    diff-lcs (1.4.4)
+    dogstatsd-ruby (3.0.0)
+    erubi (1.10.0)
+    excon (0.85.0)
+    exponential-backoff (0.0.4)
+    ffi (1.15.3)
+    fig_tree (0.0.2)
+      activesupport (>= 3.0.0)
+    flipp-ruby-dogstatsd (1.0.8)
+      activesupport (>= 3.0)
+      dogstatsd-ruby (~> 3.0.0)
+    flipp_ruby_style (0.1.4)
+      rubocop (~> 0.76)
+      rubocop-performance (~> 1.5)
+      rubocop-rails (~> 2.3)
+      rubocop-rspec (~> 1.27)
+      thor
+    formatador (0.3.0)
+    guard (2.18.0)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.13.0)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    guard-rubocop (1.4.0)
+      guard (~> 2.0)
+      rubocop (< 2.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    listen (3.6.0)
+      rb-fsevent (~> 0.10, >= 0.10.3)
+      rb-inotify (~> 0.9, >= 0.9.10)
+    little-plugger (1.1.4)
+    logging (2.3.0)
+      little-plugger (~> 1.1)
+      multi_json (~> 1.14)
+    loofah (2.12.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.5.9)
+    lumberjack (1.2.8)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    msgpack (1.4.2)
+    multi_json (1.15.0)
+    nenv (0.3.0)
+    nokogiri (1.12.3-x86_64-darwin)
+      racc (~> 1.4)
+    notiffany (0.1.3)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    parallel (1.20.1)
+    parser (3.0.2.0)
+      ast (~> 2.4.1)
+    phobos_temp_fork (0.0.1)
+      activesupport (>= 3.0.0)
+      concurrent-ruby (>= 1.0.2)
+      concurrent-ruby-ext (>= 1.0.2)
+      exponential-backoff
+      logging
+      thor
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    racc (1.5.2)
+    rack (2.2.3)
+    rack-test (1.1.0)
+      rack (>= 1.0, < 3)
+    rails-dom-testing (2.0.3)
+      activesupport (>= 4.2.0)
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.3.0)
+      loofah (~> 2.3)
+    railties (6.1.4)
+      actionpack (= 6.1.4)
+      activesupport (= 6.1.4)
+      method_source
+      rake (>= 0.13)
+      thor (~> 1.0)
+    rainbow (3.0.0)
+    rake (13.0.6)
+    rb-fsevent (0.11.0)
+    rb-inotify (0.10.1)
+      ffi (~> 1.0)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rubocop (0.93.1)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 0.6.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.10.0)
+      parser (>= 3.0.1.1)
+    rubocop-performance (1.10.2)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rails (2.9.1)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 0.90.0, < 2.0)
+    rubocop-rspec (1.44.1)
+      rubocop (~> 0.87)
+      rubocop-ast (>= 0.7.1)
+    ruby-progressbar (1.11.0)
+    shellany (0.0.1)
+    sigurd (0.0.3)
+      concurrent-ruby (~> 1)
+      exponential-backoff
+    thor (1.1.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  flipp-ruby-kafka-temp-fork!
+  flipp_ruby_style
+  guard (~> 2)
+  guard-rspec (~> 4)
+  guard-rubocop (~> 1)
+  rspec (~> 3)
+  rspec_junit_formatter (~> 0.3)
+  rubocop (~> 0.58)
+  rubocop-rspec (~> 1.27)
+
+BUNDLED WITH
+   2.2.3

data/Guardfile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { 'spec' }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml|\.slim)$})          { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { 'spec' }
+  watch('config/routes.rb')                           { 'spec/routing' }
+end
+
+guard :rubocop do
+  watch(/.+\.rb$/)
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end

data/README.md

@@ -0,0 +1,102 @@
+# FlippRubyKafka
+
+Flipp's best practices for using Kafka producers and consumers.
+Built on top of Deimos, and also integrates
+DataDog metrics and APM.  
+
+See the Deimos gem for most configuration and usage.
+
+## Installation
+
+Add this line to your application's Gemfile:
+```ruby
+gem 'flipp-ruby-kafka', require: 'flipp_ruby_kafka'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install flipp-ruby-kafka
+
+## Versioning
+
+We use semver for this gem. Any change in previous behavior is denoted
+with a bump in the minor version (0.4 -> 0.5). Patch versions are for
+bugfixes or new functionality which does not affect existing code. You
+should be locking your Gemfile to the minor version:
+
+```ruby
+gem 'flipp-ruby-kafka', '~> 0.4', require: 'flipp_ruby_kafka'
+```
+
+## Configuration
+
+FlippRubyKafka comes with configuration out of the box for the Flipp
+Microservice Platform. Your application configuration should look something
+like this (in e.g. `config/initializers/flipp_ruby_kafka.rb`):
+
+```ruby
+  FlippRubyKafka.configure do
+    producers.schema_namespace 'com.flipp.my_app'
+    # For integration testing - see below
+    kafkateria_url ENV['KAFKATERIA_URL']
+  end
+  FlippRubyKafka.configure_datadog
+```
+
+FlippRubyKafka also changes the default for a number of settings to make
+your configuration smaller in the 80% case. The following are the setting
+defaults:
+
+Config name|Default
+-----------|-------
+logger|Clone of Rails logger
+schema.backend|`:avro_validation` for development, `:avro_schema_registry` otherwise
+schema.path|`"#{Rails.root}/app/schemas"`
+schema.registry_url|`ENV['KAFKA_SCHEMA_REGISTRY'] \|\| 'http://localhost:8081'`
+consumers.report_lag|true for staging/production, false for development/test
+consumers.fatal_error|true if ActiveRecord connection is not active
+kafka.seed_brokers|`ENV['KAFKA_SSL_BROKERS'] \|\| 'localhost:9092'`
+kafka.ssl.enabled|true for staging/production, false for development/test
+kafka.ssl.ca_cert|'/var/certs/kafka-broker-ca-public-cert'
+kafka.ssl.client_cert|'/var/certs/kafka-client-public-key'
+kafka.ssl.client_cert_key|'/var/certs/kafka-client-private-key-decrypted'
+kafka.ssl.verify_hostname|false
+kafkaeria_url|'http://localhost:3000'
+
+## Running consumers
+
+FlippRubyKafka aliases `rake deimos:start` to `rake phobos:start`
+to support legacy applications.
+
+## Testing
+
+FlippRubyKafka has helper methods for sending API requests to
+[Kafkateria](https://github.com/wishabi/kafkateria):
+
+```ruby
+
+kafkateria_delete_topic('my-topic')
+
+kafkateria_produce_messages(
+  topic: 'my-topic',
+  num_messages: 5,
+  traits: %i(indexed),
+  values: { merchant_id: 1 }
+)
+```
+
+Make sure to set `config.kafkateria_url` when configuring FlippRubyKafka
+to enable this.
+
+## Additional Tools
+
+FlippRubyKafka provides generators to hook into the platform build's features. These generate JSON config files to be used by the build script:
+
+* `rails g flipp_ruby_kafka:schema_validation` will generate a `schema_validation.json`
+* `rails g flipp_ruby_kafka:topic_validation` will generate a `topic_validation.json`
+
+You can run both scripts in your build step - as long as the files are persisted to your workspace, you can use them in your deploy or in your separate validation steps.

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+  task(default: :spec)
+rescue LoadError # rubocop:disable Lint/HandleExceptions
+  # no rspec available
+end
+
+import('./lib/tasks/phobos.rake')

data/bin/flipp_ruby_kafka

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'flipp_ruby_kafka'

data/deploy/artifactory.sh

@@ -0,0 +1,8 @@
+cd ./deploy
+
+#download latest deploy script
+aws s3 cp "s3://flipp-platform-production/deploy/library-builder/library-builder-deploy.sh" library-builder-deploy.sh
+chmod +x library-builder-deploy.sh
+
+#Execute the deployment Scripts
+./library-builder-deploy.sh

data/deploy/config.json

@@ -0,0 +1,4 @@
+{
+  "type": "ruby",
+  "publishType": "artifactory"
+}

data/docker-compose.yml

@@ -0,0 +1,89 @@
+version: '3.6'
+services:
+  mysql:
+    image: mysql:5.7
+    expose:
+      - 3306
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_DATABASE=test
+      - TZ=America/Toronto
+
+  postgres:
+    image: postgres:11.1
+    expose:
+      - 5432
+    environment:
+      POSTGRES_PASSWORD: root
+
+  test:
+    volumes:
+      - .:/var/app
+    depends_on:
+      - kafka-broker
+      - mysql
+      - postgres
+      - kafkateria
+    build: .
+    environment:
+      - "DEFAULT_TIMEOUT=${DEFAULT_TIMEOUT}"
+      - MYSQL_HOST=mysql
+      - PG_HOST=postgres
+      - KAFKATERIA_URL=http://kafkateria:3000
+      - SCHEMA_REGISTRY=http://schema-registry:8081
+      - KAFKA_SEED_BROKER=kafka-broker:9092
+    command: dockerize -wait tcp://mysql:3306 -wait tcp://postgres:5432 -timeout 1m rspec
+
+  zookeeper:
+    image: wurstmeister/zookeeper:latest
+    ports:
+     - 2181:2181
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry
+    hostname: schema-registry
+    depends_on:
+      - zookeeper
+      - kafka-broker
+    ports:
+      - "8081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
+
+  kafka-broker:
+    image: confluentinc/cp-enterprise-kafka
+    hostname: kafka-broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka-broker:9092'
+      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
+      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+
+  kafkateria:
+    image: 421990735784.dkr.ecr.us-east-1.amazonaws.com/flipp/kafkateria:latest
+    hostname: kafkateria
+    ports:
+      - "3000:3000"
+    environment:
+      SCHEMA_REGISTRY: 'http://schema-registry:8081'
+      KAFKA_SEED_BROKER: 'kafka-broker:9092'
+    depends_on:
+      - kafka-broker
+      - schema-registry
+    volumes:
+      - ./kafkateria/factories:/var/app/external/factories
+      - ./spec/schemas:/var/app/external/schemas
+    command:
+      rails s -p3000