tpt-rails 1.5.0 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bdc8c4374f7b3bf47715b791d8d6f04295818b075210ef382c5f57b138f90c49
-  data.tar.gz: e809b8da64edc2b4ebebfd87f2e301475d6b6117fccb3acc0aed997b459d9218
+  metadata.gz: 300608f90255d2884190ff472e70781927269af457bf549d57443edc396186ca
+  data.tar.gz: 5dd5a558b911dd7dae7486f0984ddeac4fa8c05d5d3407e5ef269d64fe7a3e4c
 SHA512:
-  metadata.gz: c7abc446ff40916c9872b910ff44a06ac502ebe459bc65b7eea13a0ff38da1fa877e63ac3f3b1c413ed054d7b9c22290c05b941592d5e2e3908378d27b2da99e
-  data.tar.gz: 812eb1ab44fedbb0e5e42e8d5ac16eeb6e46b74421d7d962c89266dc572b2540b26feaf914547403d477ac3cc220b38ee1e35971d4ba9fb00e61c48c83322cd3
+  metadata.gz: 849e1766a9d4a6668516bb1d52d313e375bd823d2723668fb9917c218e79444e71400803cfecdea4f4ec7f8521832fea4c962d36af814b93fe1d4735e228d3c0
+  data.tar.gz: 7865a91a0e8b6553647ee022d84617aad7ef590c5e9fce5f64ea30eeeb25f7dd60ea0f71ea28702cad0f0eed20f02cd588467c657edb4610bd679437e2a63784

data/README.md

@@ -22,11 +22,13 @@ can configure this gem and enable optional features in that file.
 See the documentation in [lib/tpt/rails.rb](lib/tpt/rails.rb).
 
 For Kubernetes configuration:
+
 ```sh
 rails generate tpt:rails:kubernetes
 ```
 
 For CI/CD configuration:
+
 ```sh
 rails generate tpt:rails:continuous_integration
 ```
@@ -45,6 +47,8 @@ This gem provides the following features:
   - Adds an error test at `/internal/error-test` that reports an error and emits a metric
 - Kubernetes/Helm configuration
 - CI/CD configuration
+- EventBridge integration
+  - Adds the aws and aws eventbridge gems, and the EventBridgePublisher class
 
 See below for more details.
 
@@ -60,6 +64,29 @@ Tpt::Rails.statsd.increment('foo', tags: ['a:b', 'c:d'])
 
 See the documentation in [lib/tpt/rails.rb](lib/tpt/rails.rb).
 
+### EventBridge
+
+`Tpt::Rails::EventBridgePublisher` is the class used to emit events to EventBridge (in batches). To use:
+
+1. In an initializer, pass in the `Datadog::Statsd` object you're using to the class method ::set_metrics.
+   (Failure to do so will result in an error.) When testing, you can pass in a local Statsd, eg:
+
+   ```ruby
+   Tpt::Rails::EventBridgePublisher.set_metrics(
+      Datadog::Statsd.new('localhost', 8125)
+   )
+   ```
+
+2. By default (#new with no arguments) EventBridgePublisher will try to use the following environment
+   variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `TPT_EVENT_SOURCE`, `TPT_SHARED_EVENT_BUS_NAME`.
+
+   Additional configuration can be set as part of the constructor;
+   [see event_bridge_publisher.rb implementation](https://github.com/TeachersPayTeachers/tpt-rails/blob/ad768d3bac0c8bd28e60a48a89aed36f01c4e17b/app/models/tpt/rails/event_bridge_publisher.rb) for more details.
+
+3. Create events by instantiating or inheriting from `Tpt::Rails::ApplicationEvent`. Event types are derived from the text before
+   "Event", so ResourceEvent has the event_type "Resource". Pass in an event or an array of events to EventBridgePublisher#publish
+   and they will be published immediately. Large arrays will be batched (default 10 per batch).
+
 ### Error reporting (e.g. Bugsnag/Rollbar)
 
 If you configure `rollbar_access_token` & `rollbar_enabled`, or `bugsnag_api_key` then your project
@@ -81,10 +108,12 @@ Tpt::Rails provides the following endpoint:
 /internal/health-check
 
 Two checks are added automatically:
+
 - An ActiveRecord database check is performed if ActiveRecord is used in the app.
 - A Redis check is performed if Redis is used in the app.
 
 You may add more health checks as follows:
+
 ```ruby
 config.health_check(:foo) { 1 == 1 }
 ```
@@ -132,8 +161,8 @@ gem 'tpt-rails', path: '/your/local/path/to/tpt/rails'
 First ensure you have permissions to publish to rubygems.org.
 
 Once you've merged, check out the `master` branch and execute:
+
 ```sh
 bundle exec gem bump --push --tag --version patch # patch/minor/major/X.X.X
 bundle exec gem release
 ```
-

data/app/models/tpt/rails/application_event.rb

@@ -0,0 +1,30 @@
+module Tpt::Rails
+  class ApplicationEvent
+    include ActiveModel::Serializers::JSON
+
+    def initialize(data, metadata: {})
+      @data = data || {}
+      @metadata = metadata
+    end
+
+    def event_type
+      self.class.name.demodulize.gsub(/Event$/, '')
+    end
+
+    def ==(other)
+      data == other.data
+    end
+
+    def attributes
+      data.merge(metadata: metadata)
+    end
+
+    protected
+    attr_reader :data, :metadata
+
+    private
+    def read_attribute_for_serialization(attr)
+      attributes[attr]
+    end
+  end
+end

data/app/models/tpt/rails/event_bridge_publisher.rb

@@ -0,0 +1,133 @@
+module Tpt::Rails
+  class EventBridgePublisher
+    def initialize(
+      event_bus_name: default_event_bus_name,
+      event_source: default_event_source,
+      batch_size: 10,
+      client_options: {}
+    )
+      @event_bus_name = event_bus_name
+      @event_source = event_source
+      @batch_size = batch_size
+      @client = make_client(client_options)
+    end
+
+    def publish(events)
+      events = [events] unless events.is_a?(Array)
+
+      logger.debug "Publishing #{events.length} events to EventBridge in #{events.length/@batch_size} batch(es)"
+
+      metrics.time('tpt.event_bridge.publish') do
+        events.each_slice(@batch_size) do |batch|
+          publish_batch(batch)
+        end
+      end
+
+      events
+    end
+
+    protected
+    def publish_batch(events)
+      metrics.time('tpt.event_bridge.publish_batch') do
+        wrapped_events = wrap_events(events)
+        logger.debug({ msg: "Publishing batch of #{events.length} events to EventBridge", events: wrapped_events })
+        res = client.put_events(entries: wrapped_events)
+        handle_response(res, events)
+      end
+    rescue StandardError => e
+      logger.error({
+        msg: 'Failed to publish events',
+        events: events,
+        error: e
+      })
+      metrics.increment('tpt.event_bridge.publish_failure', by: events.length, tags: ["error_code:publish_batch_failure"])
+
+      # re-reraise the error for users to handle
+      raise e
+    end
+
+    def wrap_events(events)
+      events.map do |event|
+        {
+          source: event_source,
+          resources: [],
+          event_bus_name: event_bus_name,
+          detail: deep_camelize(event.as_json).to_json,
+          detail_type: event.event_type,
+        }
+      end
+    end
+
+    def handle_response(res, events)
+      res.data.entries.each_with_index do |entry, index|
+        if entry.error_code.present?
+          metrics.increment('tpt.event_bridge.publish_failure', tags: ["error_code:#{entry.error_code}"])
+
+          logger.error({
+            msg: "Error publishing event to EventBridge",
+            error_code: entry.error_code,
+            error_message: entry.error_message,
+            event: events[index]
+          })
+        elsif entry.event_id.present?
+          metrics.increment('tpt.event_bridge.publish_success')
+        end
+      end
+
+      res.data.entries
+    end
+
+    private
+
+    attr_reader :client, :event_bus_name, :event_source
+
+    def make_client(options)
+      all_options = {
+        access_key_id: ENV.fetch('AWS_ACCESS_KEY_ID'),
+        secret_access_key: ENV.fetch('AWS_SECRET_ACCESS_KEY'),
+        region: ENV.fetch('AWS_REGION', 'us-east-1'),
+        http_open_timeout: 1, # default is 15
+        http_read_timeout: 1, # default is 60
+      }.merge(options)
+
+      Aws::EventBridge::Client.new(all_options)
+    end
+
+    def deep_camelize(hash)
+      hash.deep_transform_keys do |k|
+        # ignore keys like "__internal" (for now)
+        if k.to_s.starts_with?("_")
+          k
+        else
+          k.to_s.camelize(:lower)
+        end
+      end
+    end
+
+    def default_event_bus_name
+      ENV.fetch('TPT_SHARED_EVENT_BUS_NAME', 'shared-event-bus-dev')
+    end
+
+    def default_event_source
+      ENV.fetch('TPT_EVENT_SOURCE')
+    end
+
+    def logger
+      Rails.logger
+    end
+
+    def self.set_metrics(statsd)
+      unless statsd.is_a?(Datadog::Statsd)
+        raise "argument passed to EventBridgePublisher::set_metrics is not a Datadog::Statsd class instance"
+      end
+
+      @@statsd = statsd
+    end
+
+    def metrics
+      raise "metrics object not set; use EventBridgePublisher#set_metrics" unless @@statsd
+
+      @@statsd
+    end
+  end
+end

data/lib/generators/tpt/rails/configuration/configuration_generator.rb

@@ -5,6 +5,6 @@ class Tpt::Rails::ConfigurationGenerator < ::Rails::Generators::Base
 
   def configure
     route "mount Tpt::Rails::Engine => '/'"
-    template 'config/initializers/tpt_rails.rb'
+    directory 'config'
   end
 end

data/lib/generators/tpt/rails/configuration/templates/config/puma.rb

@@ -0,0 +1,52 @@
+# Puma can serve each request in a thread from an internal thread pool.
+# The `threads` method setting takes two numbers: a minimum and maximum.
+# Any libraries that use thread pools should be configured to match
+# the maximum value specified for Puma. Default is set to 5 threads for minimum
+# and maximum; this matches the default thread size of Active Record.
+#
+max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }
+min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count }
+threads min_threads_count, max_threads_count
+
+# Specifies the `worker_timeout` threshold that Puma will use to wait before
+# terminating a worker in development environments.
+#
+worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development"
+
+# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
+#
+port ENV.fetch("PORT") { 3000 }
+
+# Specifies the `environment` that Puma will run in.
+#
+environment ENV.fetch("RAILS_ENV") { "development" }
+
+# Specifies the `pidfile` that Puma will use.
+pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" }
+
+before_fork do
+  Tpt::Rails::PumaStatsCollector.run(metrics_client: Tpt::Rails.statsd)
+end
+
+# Specifies the number of `workers` to boot in clustered mode.
+# Workers are forked web server processes. If using threads and workers together
+# the concurrency of the application would be max `threads` * `workers`.
+# Workers do not work on JRuby or Windows (both of which do not support
+# processes).
+
+workers ENV.fetch("WEB_CONCURRENCY") { 2 }
+
+on_worker_boot do
+  # See: https://devcenter.heroku.com/articles/deploying-rails-applications-with-the-puma-web-server#on-worker-boot
+  ActiveRecord::Base.establish_connection
+end
+
+# Use the `preload_app!` method when specifying a `workers` number.
+# This directive tells Puma to first boot the application and load code
+# before forking the application. This takes advantage of Copy On Write
+# process behavior so workers use less memory.
+
+preload_app!
+
+# Allow puma to be restarted by `rails restart` command.
+plugin :tmp_restart

data/lib/generators/tpt/rails/kubernetes/templates/kubernetes/helm/templates/deployment.yaml.tt

@@ -39,7 +39,7 @@ spec:
         tpt/creator: {{ .Values.annotations.creator | default "UNKNOWN" | quote }}
       annotations:
         buildID: "{{ .Values.buildID }}"
-        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/inject: "false"
     spec:
       # Specifies where our Kube secrets are coming from
       imagePullSecrets:

data/lib/generators/tpt/rails/kubernetes/templates/kubernetes/helm/templates/service.yaml.tt

@@ -36,6 +36,7 @@ metadata:
           - request_label:
             - frontend
 {{- if eq .Values.service.type "LoadBalancer" }}
+    external-dns.alpha.kubernetes.io/hostname: {{ .Values.hostname }}
     service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http"
     service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true"
     service.beta.kubernetes.io/aws-load-balancer-internal: "0.0.0.0/0"

data/lib/generators/tpt/rails/kubernetes/templates/kubernetes/helm/values.prod.yaml.tt

@@ -1,6 +1,8 @@
 name: <%= Tpt::Rails.app_name %>
 deploy_env: production
 
+hostname: <%= Tpt::Rails.app_name %>-production.teacherspayteachers.com
+
 annotations:
   costEnv: production
 

data/lib/generators/tpt/rails/kubernetes/templates/kubernetes/helm/values.staging.yaml.tt

@@ -1,6 +1,8 @@
 name: <%= Tpt::Rails.app_name %>
 deploy_env: staging
 
+hostname: <%= Tpt::Rails.app_name %>-staging.teacherspayteachers.com
+
 annotations:
   costEnv: staging
 

data/lib/tpt/rails/version.rb

@@ -1,6 +1,6 @@
 module Tpt
   module Rails
     # Do not change this manually. Our tooling will do it automatically.
-    VERSION = '1.5.0'
+    VERSION = '1.6.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tpt-rails
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.6.0
 platform: ruby
 authors:
 - TpT
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-18 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -58,6 +58,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.40'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.114.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.114.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-eventbridge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
@@ -88,11 +116,14 @@ files:
 - app/helpers/tpt/rails/application_helper.rb
 - app/jobs/tpt/rails/application_job.rb
 - app/mailers/tpt/rails/application_mailer.rb
+- app/models/tpt/rails/application_event.rb
 - app/models/tpt/rails/application_record.rb
+- app/models/tpt/rails/event_bridge_publisher.rb
 - app/views/layouts/tpt/rails/application.html.erb
 - config/routes.rb
 - lib/generators/tpt/rails/configuration/configuration_generator.rb
 - lib/generators/tpt/rails/configuration/templates/config/initializers/tpt_rails.rb
+- lib/generators/tpt/rails/configuration/templates/config/puma.rb
 - lib/generators/tpt/rails/continuous_integration/continuous_integration_generator.rb
 - lib/generators/tpt/rails/continuous_integration/templates/ci/Dockerfile.tt
 - lib/generators/tpt/rails/continuous_integration/templates/ci/docker-compose.yaml.tt