super_spreader 0.1.0.beta2 → 0.1.0

data/lib/super_spreader/scheduler_config.rb

@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-require "super_spreader/peak_schedule"
-require "super_spreader/redis_model"
-
-module SuperSpreader
-  class SchedulerConfig < RedisModel
-    # The job class to enqueue on each run of the scheduler.
-    attribute :job_class_name, :string
-    # The number of records to process in each invocation of the job class.
-    attribute :batch_size, :integer
-    # The amount of work to enqueue, in seconds.
-    attribute :duration, :integer
-
-    # The number of jobs to enqueue per second, allowing for fractional amounts
-    # such as 1 job every other second using `0.5`.
-    attribute :per_second_on_peak, :float
-    # The same as per_second_on_peak, but for times that are not identified as
-    # on-peak.
-    attribute :per_second_off_peak, :float
-
-    # This section manages the definition "on peak."  Compare this terminology
-    # to bus or train schedules.
-
-    # The timezone to use for time calculations.
-    #
-    # Example: "America/Los_Angeles" for Pacific time
-    attribute :on_peak_timezone, :string
-    # The 24-hour hour on which on-peak application usage starts.
-    #
-    # Example: 5 for 5 AM
-    attribute :on_peak_hour_begin, :integer
-    # The 24-hour hour on which on-peak application usage ends.
-    #
-    # Example: 17 for 5 PM
-    attribute :on_peak_hour_end, :integer
-    # The wday value on which on-peak application usage starts.
-    #
-    # Example: 1 for Monday
-    attribute :on_peak_wday_begin, :integer
-    # The wday value on which on-peak application usage ends.
-    #
-    # Example: 5 for Friday
-    attribute :on_peak_wday_end, :integer
-
-    attr_writer :schedule
-
-    def job_class
-      job_class_name.constantize
-    end
-
-    def super_spreader_config
-      [job_class, job_class.super_spreader_model_class]
-    end
-
-    def spread_options
-      {
-        batch_size: batch_size,
-        duration: duration,
-        per_second: per_second
-      }
-    end
-
-    def per_second
-      schedule.on_peak? ? per_second_on_peak : per_second_off_peak
-    end
-
-    private
-
-    def schedule
-      @schedule ||=
-        PeakSchedule.new(
-          on_peak_wday_range: on_peak_wday_begin..on_peak_wday_end,
-          on_peak_hour_range: on_peak_hour_begin..on_peak_hour_end,
-          timezone: on_peak_timezone
-        )
-    end
-  end
-end

data/lib/super_spreader/scheduler_job.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-require "active_job"
-require "json"
-require "super_spreader/scheduler_config"
-require "super_spreader/spreader"
-require "super_spreader/stop_signal"
-
-module SuperSpreader
-  class SchedulerJob < ActiveJob::Base
-    extend StopSignal
-
-    def perform
-      return if self.class.stopped?
-
-      log(started_at: Time.current.iso8601)
-      log(config.serializable_hash)
-
-      super_spreader = Spreader.new(*config.super_spreader_config)
-      next_id = super_spreader.enqueue_spread(**config.spread_options)
-      log(next_id: next_id)
-
-      return if next_id.zero?
-
-      self.class.set(wait_until: next_run_at).perform_later
-      log(next_run_at: next_run_at.iso8601)
-    end
-
-    def next_run_at
-      config.duration.seconds.from_now
-    end
-
-    def config
-      @config ||= SchedulerConfig.new
-    end
-
-    private
-
-    def log(hash)
-      SuperSpreader.logger.info({subject: self.class.name}.merge(hash).to_json)
-    end
-  end
-end

data/lib/super_spreader/spread_tracker.rb

@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require "active_record"
-require "redis"
-
-module SuperSpreader
-  class SpreadTracker
-    def initialize(job_class, model_class)
-      @job_class = job_class
-      @model_class = model_class
-    end
-
-    def initial_id
-      redis_value = redis.hget(initial_id_key, @model_class.name)
-
-      value = redis_value || @model_class.maximum(:id)
-
-      value.to_i
-    end
-
-    def initial_id=(value)
-      if value.nil?
-        redis.hdel(initial_id_key, @model_class.name)
-      else
-        redis.hset(initial_id_key, @model_class.name, value)
-      end
-    end
-
-    private
-
-    def redis
-      SuperSpreader.redis
-    end
-
-    def initial_id_key
-      "#{@job_class.name}:initial_id"
-    end
-  end
-end

data/lib/super_spreader/spreader.rb

@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-require "super_spreader/spread_tracker"
-
-module SuperSpreader
-  class Spreader
-    def initialize(job_class, model_class, spread_tracker: nil)
-      @job_class = job_class
-      @model_class = model_class
-      @spread_tracker = spread_tracker || SpreadTracker.new(job_class, model_class)
-    end
-
-    def spread(batch_size:, duration:, per_second:, initial_id:, begin_at: Time.now.utc)
-      end_id = initial_id
-      segment_duration = 1.0 / per_second
-      time_index = 0.0
-      batches = []
-
-      while time_index < duration
-        break if end_id <= 0
-
-        # Use floor to prevent subsecond times
-        run_at = begin_at + time_index.floor
-        begin_id = clamp(end_id - batch_size + 1)
-        batches << {run_at: run_at, begin_id: begin_id, end_id: end_id}
-
-        break if begin_id == 1
-
-        end_id = begin_id - 1
-        time_index += segment_duration
-      end
-
-      batches
-    end
-
-    def enqueue_spread(**opts)
-      initial_id = @spread_tracker.initial_id
-      return 0 if initial_id.zero?
-
-      batches = spread(**opts.merge(initial_id: initial_id))
-
-      batches.each do |batch|
-        @job_class
-          .set(wait_until: batch[:run_at])
-          .perform_later(batch[:begin_id], batch[:end_id])
-      end
-
-      last_begin_id = batches.last[:begin_id]
-      next_id = last_begin_id - 1
-      @spread_tracker.initial_id = next_id
-
-      next_id
-    end
-
-    private
-
-    def clamp(value)
-      (value <= 0) ? 1 : value
-    end
-  end
-end

data/lib/super_spreader/stop_signal.rb

@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-require "redis"
-
-module SuperSpreader
-  module StopSignal
-    def stop!
-      redis.set(stop_key, true)
-    end
-
-    def go!
-      redis.del(stop_key)
-    end
-
-    def stopped?
-      redis.exists(stop_key).positive?
-    end
-
-    private
-
-    def redis
-      SuperSpreader.redis
-    end
-
-    def stop_key
-      "#{name}:stop"
-    end
-  end
-end

data/lib/super_spreader/version.rb

@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-module SuperSpreader
-  VERSION = "0.1.0.beta2"
-end

data/lib/super_spreader.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-require "super_spreader/version"
-
-require "super_spreader/batch_helper"
-require "super_spreader/peak_schedule"
-require "super_spreader/redis_model"
-require "super_spreader/scheduler_config"
-require "super_spreader/scheduler_job"
-require "super_spreader/spread_tracker"
-require "super_spreader/spreader"
-require "super_spreader/stop_signal"
-
-module SuperSpreader
-  class Error < StandardError; end
-
-  class << self
-    attr_accessor :logger, :redis
-  end
-end

data/super_spreader.gemspec

@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-lib = File.expand_path("../lib", __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "super_spreader/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "super_spreader"
-  spec.version = SuperSpreader::VERSION
-  spec.authors = ["Benjamin Oakes"]
-  spec.email = ["boakes@doximity.com"]
-
-  spec.summary = "ActiveJob-based backfill orchestration library"
-  spec.description = "Provides tools for managing resource-efficient backfills of large datasets via ActiveJob"
-  spec.homepage = "https://github.com/doximity/super_spreader"
-
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "https://github.com/doximity/super_spreader/blob/main/CHANGELOG.md"
-
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  end
-  spec.executables = []
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency "activejob", ">= 6.1", "< 8.0"
-  spec.add_dependency "activemodel", ">= 6.1", "< 8.0"
-  spec.add_dependency "activerecord", ">= 6.1", "< 8.0"
-  spec.add_dependency "activesupport", ">= 6.1", "< 8.0"
-  spec.add_dependency "redis", ">= 4.8", "< 6.0"
-  spec.add_development_dependency "bundler"
-  spec.add_development_dependency "factory_bot"
-  spec.add_development_dependency "guard"
-  spec.add_development_dependency "guard-rspec"
-  spec.add_development_dependency "pry"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rspec-rails"
-  spec.add_development_dependency "rspec_junit_formatter"
-  spec.add_development_dependency "sqlite3"
-  spec.add_development_dependency "standard"
-  spec.add_development_dependency "yard"
-end