has_helpers 5.0.2

data/lib/has_helpers/data_migration/base.rb

@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+
+#== Data Migrations
+# All data migrations are processed in GoodJob. In most cases, they should
+# be idempotent. When we deploy, GoodJob will be stopped, and jobs should be
+# able to be restarted.
+#
+# To process data migrations, run `::HasHelpers::DataMigration::Migrator.run_async`.
+# This is safe to run multiple times, as jobs are unique by default.
+#
+# It is recommended to put data migration classes in db/data_migrate.
+#
+# NOTE: Currently, all data migration class names must be unique unless
+# deleted from the database. This can be resolved in a future update
+#
+#== Example
+# class UpdateUsers < ::HasHelpers::DataMigration::Base
+#   # Setting idempotent to `true` will allow the job to be restarted if it
+#   # is cancelled before finishing.
+#   self.idempotent = true
+#
+#   # Setting urgent to `true` will allow the job to be started during business
+#   # hours. By default, data migration jobs are only started at night or over
+#   # the weekend.
+#   self.urgent = true
+#
+#   # This migration will not run until OtherMigration has ran.
+#   self.dependencies = ["OtherMigration"]
+#
+#   # Only run in these environments. Defaults to ["production", "staging", "development"]
+#   self.environments = ["development", "staging"]
+#
+#   # Someone will be notified if the migration takes longer than this amount
+#   # of time (defaults to 24 hours). Data migrations with expected durations
+#   # greater than LARGE_MIGRATION_THRESHOLD are considered "large"; others
+#   # are "small." We limit goodjob to run a max of 1 large and 5 small data
+#   # migrations at a time.
+#   self.expected_duration = 4.hours
+#
+#   def run
+#     Users.find_each do |user|
+#       logger.info "Proccessing user #{user.username}"
+#       # do some work
+#     end
+#   end
+# end
+module HasHelpers
+  class DataMigration
+    class Base
+      LARGE_MIGRATION_THRESHOLD = 12.hours
+
+      def run
+        raise NotImplementedError
+      end
+
+      private
+
+      def internal_run
+        file_content = []
+        if self.class.started? && !self.class.idempotent?
+          ::Rails.logger.info "Data migration #{self.class.name} has been partially run, but is not idempotent"
+          return
+        end
+        file_content << "Data migration #{self.class.name} started at #{::Time.current}"
+        ::Rails.logger.info "Data migration #{self.class.name} started"
+        self.class.send(:start!)
+        migrate_output = run
+        file_content << migrate_output
+        self.class.send(:complete!)
+        file_content << "Data migration #{self.class.name} completed at #{::Time.current}"
+        upload_migrate_output(file_content) if self.class.const_defined?(:BUCKET_NAME) && self.class.const_defined?(:S3_FILE_PATH)
+        ::Rails.logger.info "Data migration #{self.class.name} completed"
+      end
+
+      def upload_migrate_output(file_content)
+        s3_file_path = self.class.const_get(:S3_FILE_PATH)
+        if s3_file_path.present?
+          migration_model = ::HasHelpers::DataMigration::Model.where(class_name: self.class.name).first_or_create
+          file_dir = self.class.get_s3_key(migration_model, s3_file_path, "txt")
+          # Create a tempfile
+          file = Tempfile.new(file_dir)
+          file.write(file_content)
+          file.rewind
+          # Upload the file to S3
+          s3_client = ::HasHelpers::Aws.new_s3_client(credentials_conf: ::HasHelpers::Aws.credentials_conf)
+
+          s3_client.put_object(
+            bucket: self.class.const_get(:BUCKET_NAME),
+            key: file_dir,
+            body: file
+          )
+        end
+      end
+
+      class << self
+        include ::HasHelpers::S3Utils
+
+        attr_writer :idempotent, :depends_on, :environments, :expected_duration, :urgent
+
+        def enqueue
+          model.update(enqueued_at: ::Time.current)
+          worker_klass.perform_later(self.name)
+        end
+
+        def continue_in_one_minute
+          model.update(enqueued_at: ::Time.current)
+          worker_klass.set(wait: 1.minute).perform_later(self.name)
+        end
+
+        # Clears out the timestamps for this data migration so it will run again.
+        def rerun
+          model.update(
+            enqueued_at: nil,
+            started_at: nil,
+            completed_at: nil
+          )
+        end
+
+        def s3_file
+          @s3_file ||= get_s3_file(bucket_name, s3_file_path)
+        end
+
+        def rerunable
+          raise "This migration is not rerunable." unless self.const_defined?(:BUCKET_NAME) && self.const_defined?(:S3_FILE_PATH)
+          raise "Already enqueued" unless enqueued? && started? && complete?
+          raise "File not found" if s3_file.blank?
+          self.class_eval do
+            self.urgent = true
+          end
+          enqueue
+          rename_s3_file
+        end
+
+        def s3_file_path
+          self.const_get(:S3_FILE_PATH)
+        end
+
+        def custom_format_datetime(datetime)
+          datetime = datetime.to_s unless datetime.is_a?(String)
+          parsed_datetime = Chronic.parse(datetime)
+          parsed_datetime.in_time_zone("UTC").strftime("%Y%m%d%H%M") if parsed_datetime
+        end
+
+        # Returns the S3 key for the new migration file  (rerun) or the output of the migration.
+        # s3_file_path_prefix is the prefix of the original S3 file path up to the second level directory.
+        # file_name is the name of the original file without the extension.
+        # file_name_modifier is the modifier string calculated from the updated_at timestamp.
+        # file_type is the file extension (csv or txt).
+        # example: shared/4/20220819_relationship_ids/20220819_relationship_ids_0424202.csv
+        # example: shared/4/20220819_relationship_ids/20220819_relationship_ids_042420231_output.txt
+        def get_s3_key(migration_model, s3_file_path, file_output = nil)
+          modifier = custom_format_datetime(migration_model.updated_at)
+          file_s3 = s3_file_path.split("/").last
+          file_name = file_s3.split(".")[0...-1].join(".")
+          file_type = file_output.nil? ? file_s3.split(".").last : file_output
+
+          s3_file_path_prefix = (s3_file_path.split("/").take 2).join("/")
+          s3_key = "#{s3_file_path_prefix}/#{file_name}/#{file_name}_#{modifier}"
+          s3_key += "_output" if file_output
+          s3_key += ".#{file_type}"
+          s3_key
+        end
+
+        def new_s3_file_path
+          get_s3_key(model, s3_file_path)
+        end
+
+        def new_s3_output_path
+          get_s3_key(model, s3_file_path, "txt")
+        end
+
+        def bucket_name
+          self.const_get(:BUCKET_NAME)
+        end
+
+        def rename_s3_file
+          s3_client = ::HasHelpers::Aws.new_s3_client(credentials_conf: ::HasHelpers::Aws.credentials_conf)
+          upload_object(s3_client, bucket_name, new_s3_file_path)
+        end
+
+        def upload_object(s3_client, bucket_name, object_key)
+          s3_client.put_object(
+            bucket: bucket_name,
+            key: object_key,
+            body: s3_file
+          )
+        end
+
+        def dependencies_met?
+          dependencies.each do |dependency|
+            return false unless dependency_has_completed(dependency)
+          end
+          true
+        end
+
+        def enqueued?
+          model.enqueued_at.present?
+        end
+
+        def started?
+          model.started_at.present?
+        end
+
+        def complete?
+          model.completed_at.present?
+        end
+
+        def ready?
+          if dependencies_met? && !complete? && run_in_current_environment? && valid_to_start_now?
+            if idempotent?
+              true
+            else
+              true unless started?
+            end
+          end
+        end
+
+        # Returns true if the data migration is urgent, or if we are outside
+        # business hours (using US Central Time). Business hours are defined as
+        # 7am-7pm M-F (except holidays as defined in the HasHelpers date
+        # extensions).
+        def valid_to_start_now?
+          Time.use_zone("Central Time (US & Canada)") do
+            urgent? ||
+              !Date.current.business_day? ||
+              ::Time.current.seconds_since_midnight < 7.hours || ::Time.current.seconds_until_end_of_day < 5.hours
+          end
+        end
+
+        def idempotent?
+          !!@idempotent
+        end
+
+        # In non-production environments, we always return true so data migrations
+        # run immediately.
+        def urgent?
+          !::HasUtils::Env.production? || !!@urgent
+        end
+
+        def dependencies
+          Array(@depends_on)
+        end
+
+        def expected_duration
+          @expected_duration || 24.hours
+        end
+
+        def is_large?
+          expected_duration > LARGE_MIGRATION_THRESHOLD
+        end
+
+        def worker_klass
+          is_large? ? HasHelpers::DataMigration::LargeWorker : HasHelpers::DataMigration::SmallWorker
+        end
+
+        def run_in_current_environment?
+          environments.include?(::HasUtils::Env.current_environment)
+        end
+
+        def environments
+          @environments.present? ? Array(@environments).map(&:to_s) : [::HasUtils::Env::DEVELOPMENT,
+                                                                       ::HasUtils::Env::PREVIEW,
+                                                                       ::HasUtils::Env::PRODUCTION,
+                                                                       ::HasUtils::Env::STAGING,
+                                                                       ::HasUtils::Env::TEST]
+        end
+
+        private
+
+        def start!
+          model.update(started_at: ::Time.current)
+        end
+
+        def complete!
+          model.update(completed_at: ::Time.current)
+        end
+
+        def model
+          ::HasHelpers::DataMigration::Model.where(class_name: self.name).first_or_create
+        end
+
+        # Returns true if the migration class name passed in has completed; false
+        # otherwise.
+        def dependency_has_completed(migration_class_name)
+          migration = ::HasHelpers::DataMigration::Model.find_by(class_name: migration_class_name)
+          migration && migration.completed_at
+        end
+      end
+
+      class Error < StandardError; end
+    end
+  end
+end

data/lib/has_helpers/data_migration/large_worker.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DataMigration
+    class LargeWorker < ::HasUtils::GoodJob::BaseWorker
+      include ::GoodJob::ActiveJobExtensions::Concurrency
+
+      queue_as :data_migrations_large
+
+      good_job_control_concurrency_with(
+        key: "data_migration-large",
+        perform_limit: 1
+      )
+
+      def perform(class_name)
+        class_name.constantize.new.send(:internal_run)
+      end
+    end
+  end
+end

data/lib/has_helpers/data_migration/master_worker.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DataMigration
+    class MasterWorker < ::HasUtils::GoodJob::BaseWorker
+      include ::HasUtils::GoodJob::UniqueJob
+
+      self.unique_for = 1.hour
+
+      def perform
+        ::HasHelpers::DataMigration::Migrator.run_async
+        check_migrations
+      end
+
+      private
+
+      def check_migrations
+        ::HasHelpers::DataMigration::Model.where.not(started_at: nil).where(completed_at: nil).find_each do |migration|
+          if migration.klass.blank?
+            ::Rails.logger.info "Data migration #{migration.class_name} no longer exists."
+            migration.destroy
+            next
+          end
+
+          if migration.exceeded_expected_duration?
+            ::Rails.logger.info "Data migration #{migration.class_name} exceeded expected duration."
+            # TODO: Notify support
+          end
+        end
+      end
+    end
+  end
+end

data/lib/has_helpers/data_migration/migrator.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DataMigration
+    class Migrator
+      def self.run_async
+        non_queued_classes.each do |migration|
+          if migration.ready?
+            migration.enqueue
+          end
+        end
+      end
+
+      # 1. By default, the rake task will try to run all pending (i.e. not
+      #    completed) data migrations synchronously, one at a time. If you set
+      #    MIGRATION, it will try just that one data migration. It will skip any
+      #    data migrations that have already been enqueued. A data migration is
+      #    enqueued when good job picks it up, or if you ran it with this rake task
+      #    and it did not complete successfully.
+      # 2. To rerun a data migration that was previously enqueued, first go into a
+      #    rails console, and execute <data migration class name>.rerun (for
+      #    example, FixMetadataCreatedByIds.rerun) - and then run this rake task.
+      # 3. If a data migration raises an uncaught error, the rake task will stop.
+      #    It will not run subsequent data migrations, BUT... since the one that
+      #    errored out is now marked as enqueued, running the rake task again will
+      #    skip that one.
+      # 4. If you have dependent data migrations, you may need to run this multiple
+      #    times. For example, if A depends on B, it’s possible the first time you
+      #    run the rake task, it will try to run A first and skip it because its
+      #    dependency has not run yet. (You could also get lucky and the rake task
+      #    might try to run B before A, but don’t count on it.) The rake task will
+      #    output info to the console on which data migrations were run versus
+      #    skipped.
+      def self.run(migration_name = nil)
+        puts "*** Starting..."
+        if migration_name
+          data_migration = migration_name.safe_constantize
+          unless data_migration && data_migration < ::HasHelpers::DataMigration::Base
+            puts "*** #{migration_name} is not a valid DataMigration."
+            return
+          end
+          if data_migration.complete?
+            puts "*** #{migration_name} is already complete"
+            return
+          end
+          run_one(data_migration)
+        else
+          non_queued_classes.each do |migration|
+            run_one(migration)
+          end
+        end
+
+        puts "*******************"
+        puts "DONE running data migrations. Check log/data_migrations.log and log/data_migrations/ for details."
+        puts "*******************"
+      end
+
+      def self.run_one(migration)
+        if migration.enqueued?
+          puts "WARNING: Skipping #{migration.name} because it is already enqueued."
+          return
+        end
+
+        if migration.ready?
+          puts "Running #{migration.name}..."
+          migration.send(:model).update(enqueued_at: Time.zone.now)
+          migration.new.send(:internal_run)
+          puts "Done #{migration.name}"
+        else
+          puts "WARNING: #{migration.name} not ready to run (may have dependencies)."
+        end
+      end
+
+      def self.migration_classes
+        ::HasHelpers::DataMigration::Base.descendants
+      end
+
+      def self.non_queued_classes
+        migration_classes.reject(&:enqueued?)
+      end
+    end
+  end
+end

data/lib/has_helpers/data_migration/model.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DataMigration
+    class Model < ::ActiveRecord::Base
+      self.table_name = "data_migrations"
+
+      def exceeded_expected_duration?
+        return false if started_at.blank?
+        (Time.zone.now - started_at) > klass.expected_duration
+      end
+
+      def klass
+        @klass ||= class_name.safe_constantize
+      end
+    end
+  end
+end

data/lib/has_helpers/data_migration/small_worker.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module HasHelpers
+  class DataMigration
+    class SmallWorker < ::HasUtils::GoodJob::BaseWorker
+      include ::GoodJob::ActiveJobExtensions::Concurrency
+
+      queue_as :data_migrations_small
+
+      good_job_control_concurrency_with(
+        key: "data_migration-small",
+        perform_limit: 5
+      )
+
+      def perform(class_name)
+        class_name.constantize.new.send(:internal_run)
+      end
+    end
+  end
+end

data/lib/has_helpers/data_migration.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "has_helpers/data_migration/model"
+require "has_helpers/data_migration/base"
+require "has_helpers/data_migration/migrator"
+require "has_helpers/data_migration/master_worker"
+require "has_helpers/data_migration/small_worker"
+require "has_helpers/data_migration/large_worker"

data/lib/has_helpers/engine.rb

@@ -0,0 +1,47 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "ostruct"
+require "searchkick"
+require_relative "../../config/initializers/redis_conf.rb"
+require "has_validated_attributes"
+require "hq-graphql"
+require "aws-sdk-s3"
+require "aws-sdk-sqs"
+require "sidekiq"
+
+begin
+  require "cancan"
+rescue LoadError
+  # no-op: cancan is optional
+end
+
+module HasHelpers
+  class Engine < ::Rails::Engine
+    isolate_namespace HasHelpers
+
+    # config.mount_at = '/'
+    # # Check the gem config
+    # initializer "check config" do |app|
+    #   # make sure mount_at ends with trailing slash
+    #   config.mount_at += '/'  unless config.mount_at.last == '/'
+    # end
+
+    # initializer "static assets" do |app|
+    #   app.middleware.use ::ActionDispatch::Static, "#{root}/public"
+    # end    # Enabling assets precompiling under rails 3.1
+    initializer :append_migrations do |app|
+      unless app.root.to_s.match root.to_s
+        config.paths["db/migrate"].expanded.each { |path| app.config.paths["db/migrate"] << path }
+      end
+    end
+
+    # Apply middleware to resolvers (Resources use resolvers derived from this class)
+    initializer "has_helpers.graphql_resource_middleware" do
+      ActiveSupport.on_load(:after_initialize) do
+        next unless defined?(::GraphQL::Schema::Resolver)
+        ::GraphQL::Schema::Resolver.prepend(HasHelpers::Middleware::ApplicationAccess)
+      end
+    end
+  end
+end

data/lib/has_helpers/extensions.rb

@@ -0,0 +1,17 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "has_helpers/core_ext/array"
+require "has_helpers/core_ext/enumerable"
+require "has_helpers/core_ext/date"
+require "has_helpers/core_ext/date/infinity"
+require "has_helpers/core_ext/false_class"
+require "has_helpers/core_ext/float"
+require "has_helpers/core_ext/hash"
+require "has_helpers/core_ext/integer"
+require "has_helpers/core_ext/nil_class"
+require "has_helpers/core_ext/object"
+require "has_helpers/core_ext/proc"
+require "has_helpers/core_ext/string"
+require "has_helpers/core_ext/symbol"
+require "has_helpers/core_ext/true_class"

data/lib/has_helpers/factory_api.rb

@@ -0,0 +1,47 @@
+# typed: false
+# frozen_string_literal: true
+
+require "active_support/security_utils"
+require "active_support/core_ext/string"
+
+module HasHelpers
+  module FactoryApi
+    HEADER = "X-Factory-Api-Key"
+    @rails_helper_loaded = false
+
+    def self.enabled?
+      return false if Rails.env.production?
+
+      Rails.env.test? && ENV.fetch("TEST_FACTORY_API_ENABLED", "false") == "true"
+    end
+
+    def self.api_key
+      ENV["TEST_FACTORY_API_KEY"]
+    end
+
+    def self.authenticated?(provided_key)
+      key = api_key
+      return false if key.blank? || provided_key.blank?
+      return false unless key.bytesize == provided_key.bytesize
+
+      ActiveSupport::SecurityUtils.secure_compare(provided_key, key)
+    end
+
+    # Load the host application's rails_helper (and thus its factory definitions) if available.
+    # Safe to call multiple times; it will only attempt once per process.
+    def self.load_rails_helper_if_present!
+      return if @rails_helper_loaded
+      return unless defined?(::Rails)
+      return unless ::Rails.respond_to?(:env) && ::Rails.env.test?
+      return unless ::Rails.respond_to?(:root) && ::Rails.root
+
+      rails_helper_path = ::Rails.root.join("spec/rails_helper.rb")
+      return unless rails_helper_path.file?
+
+      require rails_helper_path.to_s
+      @rails_helper_loaded = true
+    rescue LoadError
+      # noop
+    end
+  end
+end

data/lib/has_helpers/factory_scenario/definition.rb

@@ -0,0 +1,33 @@
+# typed: false
+# frozen_string_literal: true
+
+module HasHelpers
+  module FactoryScenario
+    class Definition
+      attr_reader :name, :extends_name, :steps, :run_block, :include_contexts, :exposed_aliases
+
+      def initialize(name, extends_name: nil)
+        @name = name
+        @extends_name = extends_name
+        @steps = []
+        @run_block = nil
+        @include_contexts = []
+        @exposed_aliases = []
+      end
+
+      def build(factory, as: nil, traits: [], default: {}, &block)
+        @steps << {
+          factory: factory.to_sym,
+          as: (as || factory).to_sym,
+          traits: Array(traits).map(&:to_sym),
+          default: default || {},
+          link_block: block
+        }
+      end
+
+      def run(&block)
+        @run_block = block
+      end
+    end
+  end
+end