gemstack 0.2.5 → 0.3.0

data/lib/gemstack/jobs/adapters/test.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    module Adapters
+      # Records jobs instead of running them (the default in tests). See
+      # GemStack::Jobs::Testing for assertions and perform_enqueued_jobs.
+      class Test
+        attr_reader :enqueued, :performed
+
+        def initialize
+          @enqueued = []
+          @performed = []
+          @mutex = Mutex.new
+          @sequence = 0
+        end
+
+        def enqueue(payload)
+          @mutex.synchronize do
+            @sequence += 1
+            @enqueued << payload.merge("id" => @sequence, "attempts" => 0)
+            @sequence
+          end
+        end
+
+        # Runs enqueued jobs (and jobs they enqueue) until none are left, or
+        # only those matching `only`. An error a job raises is re-raised (unless
+        # the job discards it), so failing jobs fail the test. Returns the outcomes.
+        def perform_enqueued(only: nil, except_ids: [])
+          names = only && Array(only).map(&:to_s)
+          outcomes = []
+          loop do
+            payload = @mutex.synchronize do
+              index = @enqueued.index do |p|
+                (names.nil? || names.include?(p["job_class"])) && !except_ids.include?(p["id"])
+              end
+              index && @enqueued.delete_at(index)
+            end
+            break unless payload
+
+            outcome = Executor.execute(payload)
+            raise outcome.error if outcome.error && outcome.status != :discarded
+
+            @performed << payload
+            outcomes << outcome
+          end
+          outcomes
+        end
+
+        def clear
+          @mutex.synchronize do
+            @enqueued.clear
+            @performed.clear
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/executor.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module GemStack
+  module Jobs
+    # Runs one job payload and decides what happens next. Shared by every
+    # adapter, so retries, discards, failures and instrumentation behave the
+    # same everywhere.
+    #
+    # Returns an Outcome: :performed, :retry (with run_at), :discarded or :failed.
+    module Executor
+      Outcome = Struct.new(:status, :error, :run_at, :attempt, keyword_init: true)
+
+      module_function
+
+      # payload: "job_class", "args", "queue", "attempts" (runs so far), "id".
+      def execute(payload)
+        attempt = Integer(payload["attempts"] || 0) + 1
+        fields = { job_class: payload["job_class"], job_id: payload["id"], queue: payload["queue"], attempt: attempt }
+        job_class = resolve(payload["job_class"])
+        started = monotonic
+        run(job_class, payload, attempt)
+        Jobs.instrument(:performed, **fields, duration_ms: elapsed(started))
+        Outcome.new(status: :performed, attempt: attempt)
+      rescue UnknownJob => e
+        Jobs.instrument(:failed, **fields, error: e)
+        Outcome.new(status: :failed, error: e, attempt: attempt)
+      rescue StandardError => e
+        failure(job_class, e, attempt, fields.merge(duration_ms: elapsed(started)))
+      end
+
+      def run(job_class, payload, attempt)
+        job = job_class.new
+        job.job_id = payload["id"]
+        job.attempt = attempt
+        job.perform(*Arguments.load(payload["args"] || []))
+      end
+
+      def failure(job_class, error, attempt, fields)
+        if job_class.discard?(error)
+          Jobs.instrument(:discarded, **fields, error: error)
+          return Outcome.new(status: :discarded, error: error, attempt: attempt)
+        end
+
+        _max, wait = job_class.retry_decision(error, attempt)
+        if wait
+          run_at = Time.now + wait
+          Jobs.instrument(:retried, **fields, error: error, run_at: run_at)
+          Outcome.new(status: :retry, error: error, run_at: run_at, attempt: attempt)
+        else
+          Jobs.instrument(:failed, **fields, error: error)
+          Outcome.new(status: :failed, error: error, attempt: attempt)
+        end
+      end
+
+      # Only GemStack::Job subclasses can run — a queue row naming another
+      # constant is never instantiated.
+      def resolve(name)
+        klass = Object.const_get(name.to_s)
+        raise UnknownJob, "#{name} is not a GemStack::Job" unless klass.is_a?(Class) && klass < Job
+
+        klass
+      rescue NameError
+        raise UnknownJob, "unknown job class #{name.inspect}"
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      def elapsed(started) = started && ((monotonic - started) * 1000).round(2)
+    end
+  end
+end

data/lib/gemstack/jobs/testing.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "gemstack/jobs"
+
+module GemStack
+  module Jobs
+    # Test helpers (included into GemStack::TestCase by the generated test helper):
+    #
+    #   def test_signup_sends_welcome_email
+    #     post_json "/api/signups", { email: "a@b.c" }
+    #
+    #     assert_enqueued SendWelcomeEmail, args: [User.last.id]
+    #     perform_enqueued_jobs
+    #     assert_equal 1, Mailer.deliveries.size
+    #   end
+    module Testing
+      def self.included(base)
+        base.class_eval do
+          def before_setup
+            super
+            GemStack::Jobs.adapter = GemStack::Jobs::Adapters::Test.new
+          end
+        end
+      end
+
+      def enqueued_jobs = Jobs.adapter.enqueued
+
+      # Jobs of job_class (optionally with these args / on this queue) were enqueued.
+      def assert_enqueued(job_class, args: nil, queue: nil, count: nil)
+        matching = enqueued_jobs.select do |job|
+          job["job_class"] == job_class.name && (args.nil? || job["args"] == Arguments.dump(args)) &&
+            (queue.nil? || job["queue"] == queue.to_s)
+        end
+        message = "Expected #{job_class.name}#{" with #{args.inspect}" if args} to be enqueued; " \
+                  "enqueued: #{enqueued_jobs.map { |j| [j["job_class"], j["args"]] }.inspect}"
+        count ? assert_equal(count, matching.size, message) : assert(!matching.empty?, message)
+      end
+
+      def refute_enqueued(job_class)
+        assert(enqueued_jobs.none? { |job| job["job_class"] == job_class.name },
+               "Expected no #{job_class.name} to be enqueued")
+      end
+
+      # Runs enqueued jobs (and any they enqueue). With a block: only jobs
+      # enqueued inside it. Errors raised by jobs propagate.
+      def perform_enqueued_jobs(only: nil)
+        return Jobs.adapter.perform_enqueued(only: only) unless block_given?
+
+        before = enqueued_jobs.map { |job| job["id"] }
+        yield
+        Jobs.adapter.perform_enqueued(only: only, except_ids: before)
+      end
+    end
+  end
+end

data/lib/gemstack/jobs/worker.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "socket"
+
+module GemStack
+  module Jobs
+    # Processes jobs from the PostgreSQL queue (`gemstack jobs`).
+    #
+    # - `concurrency` threads each claim and run one job at a time.
+    # - A listener thread LISTENs for NOTIFY and wakes idle threads at once;
+    #   idle threads also re-check every poll_interval.
+    # - A reaper releases jobs whose lock is older than lock_timeout (a worker
+    #   crashed mid-job), so they run again (at-least-once delivery).
+    # - stop (SIGINT/SIGTERM) lets running jobs finish for shutdown_timeout,
+    #   then releases the rest back to the queue.
+    #
+    # Needs concurrency + 2 database connections (the CLI sizes the pool).
+    class Worker
+      attr_reader :id
+
+      def initialize(store: Adapters::Database.new, queues: Jobs.config.queues, concurrency: Jobs.config.concurrency,
+                     poll_interval: Jobs.config.poll_interval, lock_timeout: Jobs.config.lock_timeout,
+                     shutdown_timeout: Jobs.config.shutdown_timeout)
+        @store = store
+        @queues = Array(queues).map(&:to_s)
+        @concurrency = concurrency
+        @poll_interval = poll_interval || (store.respond_to?(:postgres?) && store.postgres? ? 5 : 1)
+        @lock_timeout = lock_timeout
+        @shutdown_timeout = shutdown_timeout
+        @id = "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.hex(3)}"
+        @mutex = Mutex.new
+        @wakeup = ConditionVariable.new
+        @running = false
+        @threads = []
+      end
+
+      def running? = @running
+
+      # Starts the threads and returns immediately.
+      def start
+        @running = true
+        GemStack.logger.info("jobs worker started", worker: @id, queues: @queues.join(","), concurrency: @concurrency)
+        @threads = Array.new(@concurrency) { |i| Thread.new { work_loop(i) } }
+        @listener = Thread.new { listen_loop } if @store.respond_to?(:postgres?) && @store.postgres?
+        @reaper = Thread.new { reap_loop }
+        self
+      end
+
+      # Blocks until SIGINT/SIGTERM, then shuts down gracefully.
+      def run
+        %w[INT TERM].each { |signal| trap(signal) { @running = false } }
+        start
+        sleep 0.2 while @running
+        shutdown
+      end
+
+      def stop
+        @running = false
+        wake
+      end
+
+      def shutdown
+        stop
+        deadline = monotonic + @shutdown_timeout
+        @threads.each { |thread| thread.join([deadline - monotonic, 0].max) }
+        unfinished = @threads.count(&:alive?)
+        @threads.each(&:kill)
+        released = @store.release(@id)
+        [@listener, @reaper].compact.each { |thread| thread.kill.join(1) }
+        GemStack.logger.info("jobs worker stopped", worker: @id, released: released, unfinished: unfinished)
+      end
+
+      # Wakes idle threads (NOTIFY arrived, or shutting down).
+      def wake = @mutex.synchronize { @wakeup.broadcast }
+
+      # Runs one job if one is ready. Returns the outcome, or nil when idle.
+      def work_once
+        payload = @store.claim(@queues, @id) or return nil
+
+        outcome = Executor.execute(payload)
+        settle(payload, outcome)
+        outcome
+      end
+
+      private
+
+      def work_loop(_index)
+        while @running
+          begin
+            idle(@poll_interval) unless work_once
+          rescue Sequel::DatabaseConnectionError, Sequel::PoolTimeout => e
+            GemStack.logger.warn("jobs worker: database unavailable, retrying", error: e.message)
+            idle(@poll_interval)
+          rescue StandardError => e # a bug in the worker itself; keep the thread alive
+            GemStack.logger.error("jobs worker error", error: e, backtrace: Array(e.backtrace).first(10))
+            idle(@poll_interval)
+          end
+        end
+      end
+
+      def settle(payload, outcome)
+        case outcome.status
+        when :performed, :discarded then @store.complete(payload["id"])
+        when :retry
+          @store.reschedule(payload["id"], run_at: outcome.run_at, attempts: outcome.attempt, error: outcome.error)
+        when :failed then @store.fail(payload["id"], attempts: outcome.attempt, error: outcome.error)
+        end
+      end
+
+      def idle(seconds)
+        @mutex.synchronize { @wakeup.wait(@mutex, seconds) if @running }
+      end
+
+      def listen_loop
+        @store.db.listen(Adapters::Database::CHANNEL, loop: ->(_conn) { throw :stop unless @running },
+                                                      timeout: @poll_interval) { wake }
+      rescue Sequel::DatabaseConnectionError => e
+        GemStack.logger.warn("jobs worker: LISTEN failed, falling back to polling", error: e.message)
+      end
+
+      def reap_loop
+        interval = [@lock_timeout / 4.0, 1].max
+        while @running
+          begin
+            released = @store.release_stale(@lock_timeout)
+            GemStack.logger.warn("jobs: released stale locks", count: released) if released.positive?
+          rescue Sequel::Error => e
+            GemStack.logger.warn("jobs reaper error", error: e.message)
+          end
+          sleep interval
+        end
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/gemstack/jobs.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+require "gemstack/core"
+
+module GemStack
+  # Background jobs (ARCHITECTURE §9).
+  #
+  #   class SendWelcomeEmail < GemStack::Job
+  #     queue :mailers
+  #     retry_on Net::ReadTimeout, attempts: 5
+  #     def perform(user_id) = Mailer.welcome(User.find(user_id))
+  #   end
+  #
+  #   SendWelcomeEmail.perform_later(user.id)
+  #   SendWelcomeEmail.set(wait: 600).perform_later(user.id)
+  #
+  # Work is only ever asynchronous when you ask for it with perform_later.
+  # Delivery is at-least-once: make perform idempotent.
+  module Jobs
+    class Config < Settings
+      # :database (default with gemstack/db loaded: the app's PostgreSQL, MySQL or
+      # SQLite; :postgres is an alias), :async (in-process threads),
+      # :inline (run immediately), :test (record only; default in tests),
+      # :sidekiq, or an adapter object responding to #enqueue(payload).
+      setting :adapter, default: lambda {
+        if GemStack.env.test? then :test
+        elsif defined?(GemStack::DB) then :database
+        else :async
+        end
+      }
+      setting :default_queue, default: "default"
+      setting :default_priority, default: 100 # lower runs first
+      setting :default_max_attempts, default: 10
+      # Worker settings (`gemstack jobs`).
+      setting :queues, default: -> { ENV.fetch("GEMSTACK_JOB_QUEUES", "*").split(",").map(&:strip) }
+      setting :concurrency, default: -> { Integer(ENV.fetch("GEMSTACK_JOB_CONCURRENCY", 5)) }
+      # Seconds between polls. PostgreSQL NOTIFY wakes workers instantly, so
+      # there polling is only a safety net (5 s); MySQL and SQLite rely on it (1 s).
+      setting :poll_interval, default: nil
+      # A job locked longer than this is assumed abandoned (worker crashed) and is released.
+      # Jobs that legitimately run longer must raise it.
+      setting :lock_timeout, default: 30 * 60
+      # How long a stopping worker waits for running jobs before releasing them.
+      setting :shutdown_timeout, default: 25
+      setting :table, default: :gemstack_jobs
+      # Keep exhausted jobs (failed_at set) for inspection and `gemstack jobs:retry`.
+      setting :keep_failed, default: true
+    end
+
+    autoload :Worker, "gemstack/jobs/worker"
+    autoload :Testing, "gemstack/jobs/testing"
+
+    # Raised for arguments that can't round-trip through JSON.
+    class SerializationError < Error; end
+
+    # A job's name didn't resolve to a GemStack::Job subclass when it ran.
+    class UnknownJob < Error; end
+
+    Event = Struct.new(:name, :job_class, :job_id, :queue, :attempt, :duration_ms, :error, :run_at, keyword_init: true)
+
+    @subscribers = Hash.new { |hash, key| hash[key] = [] }
+    @mutex = Mutex.new
+
+    class << self
+      def config = GemStack.config.jobs
+
+      def adapter
+        @adapter || @mutex.synchronize { @adapter ||= build_adapter(config.adapter) }
+      end
+
+      attr_writer :adapter
+
+      def build_adapter(setting)
+        case setting
+        when :database, "database", :postgres, "postgres" then Adapters::Database.new
+        when :async, "async" then Adapters::Async.new
+        when :inline, "inline" then Adapters::Inline.new
+        when :test, "test" then Adapters::Test.new
+        when :sidekiq, "sidekiq" then Adapters::Sidekiq.new
+        else
+          raise ConfigurationError, "a job adapter must respond to #enqueue" unless setting.respond_to?(:enqueue)
+
+          setting
+        end
+      end
+
+      # True when jobs are rows in the application's database (and need a worker).
+      def database_queue?(setting = config.adapter) = %w[database postgres].include?(setting.to_s)
+
+      # Instrumentation for metrics/monitoring:
+      #   GemStack::Jobs.subscribe(:failed) { |event| Sentry.capture_message(...) }
+      # Events: :enqueued, :performed, :retried, :failed, :discarded.
+      def subscribe(name = :all, &block)
+        @mutex.synchronize { @subscribers[name.to_sym] << block }
+        block
+      end
+
+      def unsubscribe(block)
+        @mutex.synchronize { @subscribers.each_value { |list| list.delete(block) } }
+      end
+
+      def instrument(name, **fields)
+        event = Event.new(name: name, **fields)
+        log(event)
+        (@subscribers[name] + @subscribers[:all]).each do |subscriber|
+          subscriber.call(event)
+        rescue StandardError => e
+          GemStack.logger.error("job subscriber failed", error: e)
+        end
+        event
+      end
+
+      def reset!
+        @adapter = nil
+      end
+
+      private
+
+      def log(event)
+        fields = { job: event.job_class, id: event.job_id, queue: event.queue, attempt: event.attempt,
+                   ms: event.duration_ms, run_at: event.run_at&.utc&.iso8601 }.compact
+        fields[:error] = "#{event.error.class}: #{event.error.message}" if event.error
+        level = { failed: :error, retried: :warn }.fetch(event.name, :info)
+        GemStack.logger.public_send(level, "job.#{event.name}", **fields)
+      end
+    end
+  end
+
+  class << self
+    def jobs = Jobs.adapter
+  end
+end
+
+require_relative "job"
+require_relative "jobs/executor"
+require_relative "jobs/adapters/inline"
+require_relative "jobs/adapters/test"
+require_relative "jobs/adapters/async"
+require_relative "jobs/adapters/database"
+require_relative "jobs/adapters/sidekiq"
+
+GemStack::Config.namespace(:jobs, GemStack::Jobs::Config)

data/lib/gemstack/logger.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module GemStack
+  # Structured, thread-safe logger.
+  #
+  #   logger.info("request", method: "GET", path: "/api/products", status: 200)
+  #   logger.debug { "expensive #{computation}" }
+  #   logger.with(request_id: id).warn("slow query", ms: 812)
+  #
+  # Formats:
+  #   :pretty  12:00:01.123 INFO  request method=GET path=/api/products status=200
+  #   :json    {"time":"...","level":"info","msg":"request","method":"GET",...}
+  #
+  # Fields whose key matches a filter (e.g. "password", "token") are replaced
+  # with "[FILTERED]", recursively. It also answers the standard ::Logger
+  # methods (#info, #level=, #debug?, ...) so it can be handed to other gems.
+  class Logger
+    LEVELS = { debug: 0, info: 1, warn: 2, error: 3, fatal: 4 }.freeze
+    LABELS = { debug: "DEBUG", info: "INFO ", warn: "WARN ", error: "ERROR", fatal: "FATAL" }.freeze
+    COLORS = { debug: 90, info: 36, warn: 33, error: 31, fatal: 35 }.freeze
+    FILTERED = "[FILTERED]"
+
+    attr_reader :level, :format, :context
+
+    def initialize(output = $stdout, level: :info, format: :pretty, filter: [], context: {}, color: nil, mutex: nil)
+      @output = output
+      # Log lines must appear when written, also when stdout is a pipe (e.g.
+      # under `gemstack dev` or a process manager) where Ruby would buffer them.
+      @output.sync = true if @output.respond_to?(:sync=)
+      self.level = level
+      @format = format.to_sym
+      @filter = Array(filter).map { |f| f.to_s.downcase }
+      @context = context
+      @color = color.nil? ? output.respond_to?(:tty?) && output.tty? : color
+      @mutex = mutex || Mutex.new
+    end
+
+    def level=(value)
+      value = value.to_s.downcase.to_sym
+      raise ArgumentError, "unknown log level #{value.inspect}" unless LEVELS.key?(value)
+
+      @level = value
+    end
+
+    # A child logger that adds fields to every entry. Shares output and lock.
+    def with(**fields)
+      self.class.new(@output, level: @level, format: @format, filter: @filter, context: @context.merge(fields),
+                              color: @color, mutex: @mutex)
+    end
+
+    LEVELS.each_key do |name|
+      define_method(name) { |message = nil, **fields, &block| log(name, message, fields, &block) }
+      define_method(:"#{name}?") { enabled?(name) }
+    end
+
+    def enabled?(severity) = @output && LEVELS.fetch(severity) >= LEVELS.fetch(@level)
+
+    # ::Logger compatibility: `add(severity_int, message)`.
+    def add(severity, message = nil, progname = nil, &)
+      name = LEVELS.key(severity) || :info
+      log(name, message || progname, {}, &)
+    end
+
+    def <<(message) = info(message.to_s.chomp)
+
+    def filter(fields)
+      return fields if @filter.empty?
+
+      fields.to_h do |key, value|
+        if filtered_key?(key) then [key, FILTERED]
+        elsif value.is_a?(Hash) then [key, filter(value)]
+        else [key, value]
+        end
+      end
+    end
+
+    private
+
+    def log(severity, message, fields)
+      return true unless enabled?(severity)
+
+      message = yield if message.nil? && block_given?
+      fields = filter(@context.empty? ? fields : @context.merge(fields))
+      line = @format == :json ? json_line(severity, message, fields) : pretty_line(severity, message, fields)
+      @mutex.synchronize { @output.write(line) }
+      true
+    rescue IOError, SystemCallError
+      true # never let logging take the application down
+    end
+
+    def filtered_key?(key)
+      key = key.to_s.downcase
+      @filter.any? { |f| key.include?(f) }
+    end
+
+    def json_line(severity, message, fields)
+      entry = { time: Time.now.utc.iso8601(3), level: severity, msg: message.to_s }
+      fields.each { |key, value| entry[key] = serializable(value) }
+      "#{JSON.generate(entry)}\n"
+    end
+
+    def pretty_line(severity, message, fields)
+      label = LABELS[severity]
+      label = "\e[#{COLORS[severity]}m#{label}\e[0m" if @color
+      pairs = fields.map { |key, value| "#{key}=#{pretty_value(value)}" }
+      [Time.now.strftime("%H:%M:%S.%L"), label, message, *pairs].join(" ") << "\n"
+    end
+
+    def pretty_value(value)
+      case value
+      when String then value.match?(/[\s"=]/) ? value.inspect : value
+      when nil then "nil"
+      when Hash, Array then JSON.generate(serializable(value))
+      else value.to_s
+      end
+    end
+
+    def serializable(value)
+      case value
+      when String, Integer, Float, true, false, nil then value
+      when Hash then value.transform_values { |v| serializable(v) }
+      when Array then value.map { |v| serializable(v) }
+      when Exception then { class: value.class.name, message: value.message }
+      else value.to_s
+      end
+    end
+  end
+end

data/lib/gemstack/mail/delivery_job.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "gemstack/jobs"
+
+module GemStack
+  module Mail
+    # Delivers a mailer action in the background (Delivery#deliver_later).
+    # Retried like any job; SMTP errors are transient more often than not.
+    class DeliveryJob < GemStack::Job
+      def self.queue(name = nil) = name ? super : (@queue || Mail.config.queue)
+
+      def perform(mailer_name, action, args)
+        mailer = Object.const_get(mailer_name)
+        raise ArgumentError, "#{mailer_name} is not a GemStack::Mailer" unless mailer.is_a?(Class) && mailer < Mailer
+
+        Mailer::Delivery.new(mailer, action, args).deliver_now
+      end
+    end
+  end
+end

data/lib/gemstack/mail/testing.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "gemstack/mail"
+
+module GemStack
+  module Mail
+    # Test helpers (included into GemStack::TestCase by `gemstack add auth`):
+    #
+    #   assert_emails(1) { post_json "/api/auth/password/forgot", { email: user.email } }
+    #   assert_equal [user.email], last_email.to
+    module Testing
+      def before_setup
+        super
+        Mail.deliveries.clear
+      end
+
+      def deliveries = Mail.deliveries
+      def last_email = Mail.deliveries.last
+
+      def assert_emails(count, &block)
+        before = Mail.deliveries.size
+        adapter = defined?(GemStack::Jobs) && GemStack::Jobs.adapter
+        adapter = nil unless adapter.respond_to?(:perform_enqueued)
+        queued = adapter ? adapter.enqueued.map { |job| job["id"] } : []
+        block&.call
+        # deliver_later mail sent inside the block counts too.
+        adapter&.perform_enqueued(only: ["GemStack::Mail::DeliveryJob"], except_ids: queued)
+        assert_equal count, Mail.deliveries.size - before,
+                     "Expected #{count} email(s), got #{Mail.deliveries.size - before}"
+      end
+    end
+  end
+end