flipp-ruby-kafka-temp-fork 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1d678755ab40d9ab99ad003a75be21d785e4d5e8c78cc1028bd93856dd534d49
+  data.tar.gz: ec7e3833b40568f8400594c198b41adc92f241eed8d4e964ad8971e9ab8b83c9
+SHA512:
+  metadata.gz: ee2897cdc8062db671e488fd35391ec16eb43de95a4c6dcd5f3cba764396987cc3d34413670dd360d076dcbe1451332efc7fa68a4b696e64efb709537ed933f6
+  data.tar.gz: e2129786ccf077e509e51398ceb50b578596679c8c302285fe4268a64be2554a02f21740eaf7f0b799d9ab93f9823638ead5b4dc0848b63def6e5335f2255714

data/.circleci/config.yml

@@ -0,0 +1,97 @@
+defaults: &defaults
+    parallelism: 1
+    working_directory: ~/workspace
+    docker:
+      - image: 421990735784.dkr.ecr.us-east-1.amazonaws.com/ci-build-environment:ruby-2.5
+        environment:
+          MYSQL_HOST: 127.0.01
+          PG_HOST: 127.0.0.1
+      - image: circleci/mysql:5.7
+        environment:
+          MYSQL_ALLOW_EMPTY_PASSWORD: true
+          MYSQL_DATABASE: test
+      - image: circleci/postgres:11-alpine-ram
+        environment:
+          POSTGRES_PASSWORD: root
+
+version: 2
+jobs:
+  build:
+    <<: *defaults
+    steps:
+      - checkout
+      - run:
+          name: Which bundler?
+          command: bundle -v
+      - run:
+          name: Install aws cli
+          command: sudo apt update && sudo apt-get install -y awscli
+      - run: aws s3 cp s3://flipp-platform-production/platform-param-decrypt/platform-param-decrypt_linux platform-param-decrypt_linux
+      - run: chmod +x platform-param-decrypt_linux
+      - run: mkdir vendor
+      - run: sudo chmod 1777 -R /home/circleci/.bundle /tmp
+      - run: ./platform-param-decrypt_linux -bundleConfig > ~/.bundle/config
+      - run: gem install bundler -v2.1 # 2.2.17 breaks https://github.com/rubygems/rubygems/issues/4581
+
+      # Restore bundle cache & npm cache
+      - restore_cache:
+          key: 'rails-{{ checksum "Gemfile.lock" }}'
+
+      # Bundle install dependencies in /tmp/
+      # so Dockerfile does not copy them since
+      # its base image is different than CircleCI
+      - run:
+          name: Bundle install
+          command: bundle install --path vendor/bundle --jobs=4 --retry=3
+
+      # Store bundle cache
+      - save_cache:
+          key: 'rails-{{ checksum "Gemfile.lock" }}'
+          paths:
+            - ~/workspace/vendor/bundle
+
+      - persist_to_workspace:
+          # Must be an absolute path, or relative path from working_directory
+          root: ~/workspace
+          # Must be relative path from root
+          paths:
+            - .
+  test-rspec:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run: gem install bundler -v2.1
+      - run:
+           name: Point bundle to vendor/bundle
+           command: bundle --path vendor/bundle
+      - run: mkdir result
+      - run:
+          name: Running rspec
+          command: dockerize -wait tcp://127.0.0.1:3306 -wait tcp://127.0.0.1:5432 -timeout 1m bundle exec rspec --format progress --format RspecJunitFormatter -o result/rspec.xml
+      - store_test_results:
+          path: ~/workspace/result
+
+  publish-to-artifactory:
+    working_directory: ~/workspace
+    docker:
+      - image: 421990735784.dkr.ecr.us-east-1.amazonaws.com/ci-build-environment:ruby-2.5
+    steps:
+      - checkout
+      - run: chmod +x ./deploy/artifactory.sh && ./deploy/artifactory.sh
+
+workflows:
+  version: 2
+  build-test-and-publish:
+    jobs:
+      - build
+      - test-rspec:
+          requires:
+            - build
+      - publish-to-artifactory:
+          filters:
+            branches:
+              only:
+                - master
+          requires:
+            - test-rspec

data/.dockerignore

@@ -0,0 +1,2 @@
+log/
+tmp/

data/.gitignore

@@ -0,0 +1,41 @@
+# Created by .ignore support plugin (hsz.mobi)
+### Ruby template
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+/log/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+test.sqlite3
+
+.idea

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper --tag ~integration --format documentation

data/.rubocop.yml

@@ -0,0 +1,56 @@
+inherit_gem:
+  flipp_ruby_style: rubocop.yml
+
+AllCops:
+  Exclude:
+    - 'lib/flipp_ruby_kafka/monkey_patches/*.rb'
+
+RSpec/ExpectActual:
+  Enabled: false
+
+RSpec/BeforeAfterAll:
+  Enabled: false
+
+Rails/ApplicationRecord:
+  Enabled: false
+
+Rails/TimeZone:
+  Enabled: false
+
+Style/DateTime:
+  Enabled: false
+
+RSpec/ContextWording:
+  Enabled: false
+
+RSpec/MessageSpies:
+  Enabled: false
+
+RSpec/EmptyExampleGroup:
+  Exclude:
+    - "spec/spec_helper.rb"
+
+Lint/UnusedMethodArgument:
+  Exclude:
+    - 'lib/flipp_ruby_kafka/publish_backend.rb'
+
+Security/YAMLLoad:
+  Exclude:
+    - 'lib/flipp_ruby_kafka.rb'
+
+Lint/UnusedBlockArgument:
+  Exclude:
+    - 'lib/flipp_ruby_kafka/test_helpers.rb'
+
+Metrics/ParameterLists:
+  Exclude:
+    - 'lib/flipp_ruby_kafka/test_helpers.rb'
+
+Security/Eval:
+  Exclude:
+    - 'spec/spec_helper.rb'
+
+Lint/RescueException:
+  Exclude:
+    - 'spec/spec_helper.rb'
+    - 'lib/flipp_ruby_kafka/utils/executor.rb'

data/.ruby-gemset

@@ -0,0 +1 @@
+flipp-ruby-kafka

data/.ruby-version

@@ -0,0 +1 @@
+3.0.0

data/CHANGELOG.md

@@ -0,0 +1,466 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [2.4.1] - 2021-05-13
+- Change localhost to 127.0.0.1 for better use with Docker.
+
+## [2.4.0] - 2021-01-28
+- Bump version of ruby-kafka.
+
+## [2.3.2] - 2020-10-13
+- Added generators for schema and topic validation - see README.
+
+## [2.3.1] - 2020-08-11
+- Update ddtrace requirement
+
+## [2.3.0] - 2020-03-19
+- Non-beta version!
+
+## [2.3.0-beta2] - 2020-01-30
+- Fix crash when not using ActiveRecord.
+
+# [2.3.0-beta1] - 2020-01-17
+- Bump Deimos version to 1.5.
+
+## [2.2.0-beta5] - 2019-12-17
+- Bugfix for previous release.
+
+## [2.2.0-beta4] - 2019-12-16
+- Clone loggers on startup.
+
+## [2.2.0-beta2] - 2019-12-16
+- Bump Deimos version to 1.4.
+
+## [2.1.4] - 2019-12-04
+- Bugfix related to phobos_config_file being set incorrectly.
+
+## [2.1.2] - 2019-11-26
+- Bugfix related to the fatal_error block for the DB
+
+## [2.1.0] - 2019-09-12
+- Fail forever if the DB is down when getting an error consuming a message.
+
+## [2.0.1] - 2019-09-11
+- Change required Deimos version to ~> 1
+
+## [2.0.0] - 2019-09-03
+- Official release of FlippRubyKafka 2.0!
+
+## [2.0.0-beta21] - 2019-08-29
+- Update Deimos beta version
+
+## [2.0.0-beta20] - 2019-08-28
+- Fix bug where scripts outside the main Rails directory wouldn't start up
+
+## [2.0.0-beta19] - 2019-08-28
+- Update Deimos beta version
+
+## [2.0.0-beta18] - 2019-08-26
+- Update Deimos beta version
+
+## [2.0.0-beta17] - 2019-08-22
+- Update Deimos beta version
+
+## [2.0.0-beta16] - 2019-08-20
+- Fix for test helpers being included before the gem itself.
+
+## [2.0.0-beta15] - 2019-08-09
+- Update Deimos beta version
+
+## [2.0.0-beta14] - 2019-08-08
+- Update Deimos beta version
+
+## [2.0.0-beta13] - 2019-08-07
+- Update Deimos beta version
+
+## [2.0.0-beta12] - 2019-08-06
+- Update Deimos beta version
+
+## [2.0.0-beta11] - 2019-08-02
+- Update Deimos beta version
+
+## [2.0.0-beta10] - 2019-08-01
+- Update Datadog configuration to be separate from Railtie.
+- Removed phobos:start rake task.
+
+## [2.0.0-beta3] - 2019-06-12
+- Change over all functionality to use the open-source Deimos project.
+
+## [1.0.0-beta19] - 2019-07-19
+- Add Kafkateria support for trait parameters
+
+## [1.0.0-beta14] - 2019-05-27
+- Another fix for DB backend for encoded partition keys. -_-
+
+## [1.0.0-beta13] - 2019-05-27
+- Another fix for DB backend for encoded keys.
+
+## [1.0.0-beta12] - 2019-05-23
+- Fix DB backend so it saves messages correctly in binary format.
+
+## [1.0.0-beta11] - 2019-05-06
+- Return the produced messages from latest Kafkateria version with `kafkateria_produce_messages`.
+
+## [1.0.0-beta10] - 2019-04-17
+- Fix crash when using InlineConsumer with a config_class
+- Proactively raise an error when calling `get_messages_for` without enough configuration
+
+## [1.0.0-beta9] - 2019-04-16
+- Updated TestHelper to be able to send null payloads
+
+## [1.0.0-beta8] - 2019-04-03
+- Fixed the TestHelpers fix correctly -_-
+
+## [1.0.0-beta7] - 2019-03-28
+- FIX: TestHelpers was stubbing out Datadog tracing in a way that didn't
+  preserve blocks that were passed in (e.g. for rendering ActionMailer).
+
+## [1.0.0-beta6] - 2019-03-012
+- BREAKING CHANGE - added config.report_lag to make lag reporting optional.
+  Default is now FALSE.
+
+## [1.0.0-beta5] - 2019-03-08
+- Fixed bug with KafkaMessage not saving
+
+## [1.0.0-beta4] - 2019-03-08
+- Changed Kafka message DB tables to use blobs instead of text.
+
+## [1.0.0-beta3] - 2019-03-07
+- A number of fixes for Rails 3
+- Fixes based on integration testing
+
+## [1.0.2-beta] - 2019-02-14
+- Removed metadata monkey patch for RubyKafka as it's no longer needed.
+
+## [1.0.1-beta] - 2019-01-21
+- Major breaking change!
+  - Introduce new key config of "none: true" which **must be used** if not
+  passing any keys to the producer.
+  - Removal of KafkaPublishFailure.
+  - Change KafkaSource to send messages *inside* the transaction instead of
+    using after_commit.
+  - In TestHelpers, changed logic so that producers and consumers are only
+    stubbed when `stub_producers_and_consumers!` is called manually.
+    This allows for easier management of integration tests.
+  - Removed checking of `:kafka_integration` metadata key.
+- Additional features:
+  - Introducing new concept of "backends", including a database backend.
+    - Add `publish_backend` to Configuration
+    - Add `start_db_backend!` to base class
+    - Add `force_send: true` option to `Producer#publish_list`, 
+       `ActiveRecordProducer.send_event` and `ActiveRecordProducer.send_events`
+    - Start a Kafka sending thread when `publish_backend` is set to `db` unless
+      configured not to
+
+## [0.7.14] - 2019-04-17
+- Fix crash when using InlineConsumer with a config_class
+- Proactively raise an error when calling `get_messages_for` without enough configuration
+
+## [0.7.13] - 2019-04-16
+- Updated TestHelper to be able to send null payloads
+
+## [0.7.12] - 2019-04-03
+- Fixed the TestHelpers fix correctly -_-
+
+## [0.7.11] - 2019-03-28
+- FIX: TestHelpers was stubbing out Datadog tracing in a way that didn't
+  preserve blocks that were passed in (e.g. for rendering ActionMailer).
+
+## [0.7.10] - 2019-03-07
+- FIX: InlineConsumer was caching the decoder when asked for multiple
+  topics in a row
+
+## [0.7.9.1] - 2019-03-07
+- FIX: Raise an error when Kafkateria returns a non-200 response code in
+  Kafkateria helper methods.
+
+## [0.7.8] - 2019-03-07
+- Added `kafkateria_get_messages` helper method.
+
+## [0.7.7] - 2019-02-28
+- Fix for kafkateria_delete_topic method.
+
+## [0.7.6] - 2019-02-11
+- Added Kafkateria helper methods to TestHelpers.
+
+## [0.7.5] - 2019-02-11
+- Added `:kafka_integration` metadata to disable stubbing of producers/consumers.
+
+## [0.7.4] - 2019-02-09
+- Added Utils::InlineConsumer for integration testing.
+
+## [0.7.2] - 2019-01-19
+- Added watched_attribute override functionality for producers.
+
+## [0.7.1] - 2019-01-03
+- Official 0.7 release.
+
+## [0.7.1-beta10] - 2018-12-20
+- Fixed issue with schema coercer not coercing values to boolean.
+
+## [0.7.1-beta9] - 2018-12-12
+- Monkey patched Phobos bug on startup.
+
+## [0.7.1-beta8] - 2018-12-07
+- Fixed bug where attributes sent to generate_payloads_for_resend() were not
+  with indifferent access.
+
+## [0.7.1-beta7] - 2018-12-07
+- Forgot to merge master -_-
+
+## [0.7.1-beta6] - 2018-12-05
+- Do not save publish failures if the failure is not related to Kafka producing
+- Do not raise an exception if the failure is related to Kafka producing and is
+  already caught
+
+## [0.7.1-beta5] - 2018-11-30
+- Rubocop fixes.
+
+## [0.7.1.-beta4] - 2018-11-29
+- Remove heartbeat thread logic and revert to RubyKafka in-thread heartbeating.
+
+## [0.7.1-beta3] - 2018-11-26
+- Centralize error notifications so both publish and Avro failures get sent
+  to the same notification, `produce_error`.
+
+## [0.7.1-beta2] - 2018-11-23
+- Fix KafkaPublishFailure so it sends for async messages as well.
+
+## [0.7.0-beta] - 2018-11-21
+- Bumped ruby-kafka version up to 0.7.4.
+
+## [0.6.22] - 2018-12-07
+- Fixed bug where attributes sent to generate_payloads_for_resend() were not
+  with indifferent access.
+
+## [0.6.22] - 2018-12-07
+- Fixed bug where attributes sent to generate_payloads_for_resend() were not
+  with indifferent access.
+
+## [0.6.21] - 2018-12-01
+- Fixed bug where nil payloads were being coerced into strings incorrectly instead
+  of raising an error.
+
+## [0.6.20] - 2018-11-20
+- Added publish_error DataDog metric.
+
+## [0.6.19] - 2018-11-16
+- Fixed crash on exit if Kafka client didn't exist.
+
+## [0.6.18] - 2018-11-14
+- Updated README.
+- Fixed bug when errors occurred with KafkaPublishFailure.
+
+## [0.6.17] - 2018-11-13
+- Bugfix: Send all outstanding messages when the app exits.
+- Bugfix: Using key_schema in a consumer should use AvroTurf::Messaging instead
+  of AvroTurf.
+
+## [0.6.16] - 2018-11-09
+- Bugfix: Crash when there's a problem with lag reporting
+
+## [0.6.15] - 2018-11-06
+- Bugfix: Ignore attr_protected in older versions of Rails.
+
+## [0.6.14] - 2018-11-05
+- Bugfix: Do not save updated_at and created_at for records if they're in the schema.
+
+## [0.6.13] - 2018-11-03
+- Add `FlippRubyKafka.disable_producers` method.
+
+## [0.6.12] - 2018-10-04
+- Fix for union types with ActiveRecordConsumer.
+
+## [0.6.11] - 2018-09-14
+- Fix for crash when given a timestamp which is blank on the consumer side
+
+## [0.6.10] - 2018-09-14
+- Allow KafkaSource to use multiple producers.
+- Do not crash if Avro encoding fails and reraise_consumer_errors is set to false.
+- Fix for test helper crash when failing `expect(topic).not_to have_sent(anything)`
+
+## [0.6.9] - 2018-09-07
+- Fix bug where logger was logging the non-coerced version of the payload instead
+  of what was actually sent.
+
+## [0.6.8] - 2018-09-06
+- Fix bug where nil payloads were crashing.
+
+## [0.6.7] - 2018-08-30
+- Lock RubyKafka version to 0.6 for now since 0.7 breaks our monkey patch
+- Coerce nil values to boolean for ActiveRecordProducers
+
+## [0.6.5] - 2018-08-20
+- Fix for KafkaPublishFailure generator not working with PostgreSQL.
+
+## [0.6.4] - 2018-08-16
+- Added PlatformSchemaValidation class to generate the schema validation file.
+- Fixed bug where the producer was not logging generated message ID / timestamps.
+
+## [0.6.3] - 2018-08-10
+- Updated avro-patches to avoid needing a monkey-patch
+- Fixes to TestHelpers to work with RSpec 2.
+
+## [0.6.2] - 2018-08-03
+- Fixed TestHelpers as well
+
+## [0.6.1] - 2018-08-03
+- Fixed a bug with ActiveRecordConsumer crashing with Avro-encoded keys.
+
+## [0.6.0] - 2018-07-30
+- Update to Phobos 1.8
+- decode_key no longer needed for consumers with encoded keys
+
+## [0.5.22] - 2018-07-22
+- Added the ability to specify partition keys separate from payload keys.
+
+## [0.5.21] - 2018-07-22
+- Added the KafkaSource mixin.
+- Added the PublishFailure model and generator.
+
+## [0.5.20] - 2018-07-13
+- One more issue related to heartbeat threading.
+
+## [0.5.19] - 2018-07-12
+- Fix for a different crash :(
+
+## [0.5.18] - 2018-07-12
+- Fix for a crash that can happen due to multithreading.
+
+## [0.5.17] - 2018-07-10
+- Changed heartbeat to be attached to the RubyKafka consumer instead of the
+  Phobos Executor. This should solve some of the stability problems around
+  heartbeat timeouts since we should now only be sending heartbeats from a
+  thread and not the main consumer loop.
+
+## [0.5.16] - 2018-06-27
+- Fixed an issue where encoding with key schemas would use the namespaced
+  subject in the schema registry instead of "{topic}-key".
+
+## [0.5.15] - 2018-06-20
+
+- Yet another fix related to heartbeats (only send lag on successful heartbeats)
+- Expose method to stub individual producer/consumer classes in tests.
+
+## [0.5.14] - 2018-06-11
+
+- Send a new metric, handler/time:time_delayed, which records the difference in 
+  time between when a message was sent and when it was consumed
+
+## [0.5.13] - 2018-06-01
+
+- Yet another fix related to heartbeat threads.
+- Allow lag to be reported with Rails 3.
+
+## [0.5.12] - 2018-06-08
+
+- Fixed a bug introduced with ruby-kafka 0.6.6 which causes heartbeats to fail.
+
+## [0.5.11] - 2018-06-01
+
+- Internal refactor related to heartbeat thread again.
+
+## [0.5.10] - 2018-05-30
+
+- Fixed another bug with heartbeat threads to ensure they shut down correctly.
+
+## [0.5.9] - 2018-05-29
+
+- Fixed lag reporting to work with batch consuming.
+- Fix TestHelpers to work with RSpec 2.
+
+## [0.5.8] - 2018-05-24
+
+- Added lag reporting to DataDog.
+
+## [0.5.7] - 2018-05-18
+
+- Set sync_produce to true when running Kafka consumers via rake phobos:start.
+- Set Phobos/RubyKafka to use the same JSON logger as FlippRubyKafka.
+- Updated failure message for the have_sent matcher to provide more information
+  (diff of the messages)
+
+## [0.5.6] - 2018-05-15
+
+- Fixed bug with heartbeat threads where they would not shut down in a timely
+  fashion.
+
+## [0.5.5] - 2018-05-14
+
+- Throw validation errors when the payload has extra fields, not just when
+  it's missing fields.
+- Add ActiveSupport::Notifications instrumentation.
+
+## [0.5.4] - 2018-05-08
+
+- Actually test local schemas against payloads.
+- Allow `field_config` to take a symbol or string as the field name.
+
+## [0.5.3] - 2018-05-04
+
+- Fixes for testing ActiveRecordConsumer.
+
+## [0.5.2] - 2018-05-04
+
+- Added ActiveRecordConsumer.
+
+## [0.5.1] - 2018-05-02
+
+### Fixed
+
+- Auto-coercion correctly works with union types and null payloads.
+
+## [0.5] - 2018-05-02
+
+### Changed
+
+- Some fields are now auto-coerced to types matching the schema.
+
+## [0.4.6] - 2018-04-30
+
+### Fixed
+
+- Using `FlippDatadog.time` was broken in RSpecs.
+
+## [0.4.5] - 2018-04-24
+
+### Changed
+
+- `ActiveRecordProducer.generate_payload` now doesn't care whether the keys
+  returned are strings or symbols.
+
+## [0.4.4] - 2018-04-22
+
+### Changed
+
+- Topic is now passed into `on_producer_error`.
+
+## [0.4.3] - 2018-04-22
+
+### Added
+
+- Added `on_producer_error` to configuration.
+
+## [0.4.2] - 2018-04-22
+
+### Fixed
+
+- Fix for crash when using `import` on records with an ActiveRecordProducer.
+
+## [0.4.1] - 2018-04-21
+
+### Added
+
+- Added ActiveRecordProducer.
+- Automatically add timestamps to the message if a `timestamp` field is in
+  the schema and the payload doesn't have a corresponding value.
+
+## [0.4] - 2018-04-20
+
+### Changed
+
+- Change the schema names used to reflect best practices (`{topic_name}-value`).