fiber_job 0.1.0

data/lib/fiber_job/worker.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'async/queue'
+
+module FiberJob
+  class Worker
+    def initialize(queues: nil, concurrency: nil)
+      @queues = queues || FiberJob.config.queues
+      @concurrency = concurrency || FiberJob.config.concurrency
+      @running = false
+      @managers = {}
+      @job_queues = {} # In-memory Async::Queue per Redis queue
+    end
+
+    def start
+      @running = true
+
+      Sync do |task|
+        # Initialize all queues first
+        @queues.each do |queue_name|
+          @job_queues[queue_name] = Async::Queue.new
+          queue_concurrency = FiberJob.config.concurrency_for_queue(queue_name)
+          @managers[queue_name] = ConcurrencyManager.new(max_concurrency: queue_concurrency)
+        end
+
+        # Start independent pollers for each queue
+        @queues.each do |queue_name|
+          task.async do
+            poll_redis_queue(queue_name)
+          end
+        end
+
+        # Start independent worker pools for each queue
+        @queues.each do |queue_name|
+          queue_concurrency = FiberJob.config.concurrency_for_queue(queue_name)
+          queue_concurrency.times do
+            task.async do
+              process_job_queue(queue_name)
+            end
+          end
+        end
+
+        # Global support fibers
+        task.async do
+          process_scheduled_jobs
+        end
+
+        task.async do
+          process_cron_jobs
+        end
+      end
+    end
+
+    def stop
+      @running = false
+
+      # Close all in-memory queues to signal completion to workers
+      @job_queues.each_value(&:close)
+    end
+
+    private
+
+    # Single poller fiber that fetches jobs from Redis and distributes to workers
+    # This eliminates Redis brpop contention by having only one fiber per queue accessing Redis
+    def poll_redis_queue(queue_name)
+      # Create dedicated Redis connection for this poller to avoid blocking other pollers
+      redis_conn = Queue.redis_connection
+      while @running
+        begin
+          # Use longer timeout since we're the only poller - no contention
+          job_data = Queue.pop(queue_name, timeout: 1.0, redis_conn: redis_conn)
+
+          if job_data
+            # Push to in-memory queue for worker fibers to process
+            @job_queues[queue_name].push(job_data)
+          end
+        rescue StandardError => e
+          FiberJob.logger.error "Redis polling error for queue #{queue_name}: #{e.message}"
+          sleep(1) # Brief pause on error
+        end
+      end
+    end
+
+    # Worker fibers process jobs from the fast in-memory queue
+    # Multiple workers can process concurrently without Redis contention
+    def process_job_queue(queue_name)
+      while @running
+        begin
+          # Fast in-memory dequeue operation
+          job_data = @job_queues[queue_name].dequeue
+
+          if job_data
+            # Use semaphore to control actual job execution concurrency
+            @managers[queue_name].execute do
+              execute_job(job_data)
+            end
+          end
+        rescue Async::Queue::ClosedError
+          # Queue closed during shutdown - exit gracefully
+          break
+        rescue StandardError => e
+          FiberJob.logger.error "Job processing error for queue #{queue_name}: #{e.message}"
+        end
+      end
+    end
+
+    def execute_job(job_data)
+      job_class = Object.const_get(job_data['class'])
+      job = job_class.new
+
+      job.retry_count = job_data['retry_count'] || 0
+
+      # if job.retry_count > 0
+      #   FiberJob.logger.info "Executing #{job_class} (retry #{job.retry_count}/#{job.max_retries})"
+      # end
+
+      begin
+        Timeout.timeout(job.timeout) do
+          args = (job_data['args'] || []).dup
+          args << job_data['enqueued_at'] if job_data['enqueued_at']
+          job.perform(*args)
+        end
+      rescue => e
+        handle_failure(job, job_data, e)
+      end
+    end
+
+    def execute_cron_job(job_data)
+      job_class = Object.const_get(job_data['class'])
+      job = job_class.new
+
+      begin
+        Timeout.timeout(job.timeout) do
+          job.perform
+        end
+      rescue => e
+        FiberJob.logger.error "Cron job #{job_data['class']} failed: #{e.message}"
+        FiberJob.logger.error e.backtrace.join("\n")
+
+        next_time = job_class.next_run_time
+        Cron.schedule_job(job_class, next_time)
+      end
+    end
+
+    def handle_failure(job, job_data, error)
+      if job.retry_count < job.max_retries
+        job.retry_count += 1
+        delay = job.retry_delay(job.retry_count)
+
+        retry_job_data = job_data.dup
+        retry_job_data['retry_count'] = job.retry_count
+
+        message = "#{job.class} failed: #{error.message}. "
+        message += "Retrying in #{delay.round(1)}s (attempt #{job.retry_count}/#{job.max_retries})"
+        FiberJob.logger.warn message
+
+        if job.priority_retry?
+          Queue.schedule_priority(job.queue, retry_job_data, Time.now.to_f + delay)
+        else
+          Queue.schedule(job.queue, retry_job_data, Time.now.to_f + delay)
+        end
+      else
+        FiberJob.logger.error "#{job.class} permanently failed after #{job.max_retries} retries: #{error.message}"
+        Queue.store_failed_job(job_data, error)
+      end
+    end
+
+    def process_scheduled_jobs
+      while @running
+        @queues.each do |queue_name|
+          Queue.scheduled_jobs(queue_name)
+        end
+
+        sleep(1)
+      end
+    end
+
+    def process_cron_jobs
+      while @running
+        due_jobs = Cron.due_jobs
+
+        due_jobs.each do |job_data|
+          queue_name = job_data['queue']
+
+          next unless @queues.include?(queue_name)
+
+          @managers[queue_name].execute { execute_cron_job(job_data) }
+        end
+
+        sleep(1)
+      end
+    end
+  end
+end

data/lib/fiber_job.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative 'fiber_job/version'
+require_relative 'fiber_job/config'
+require_relative 'fiber_job/logger'
+require_relative 'fiber_job/job'
+require_relative 'fiber_job/queue'
+require_relative 'fiber_job/worker'
+require_relative 'fiber_job/concurrency'
+require_relative 'fiber_job/process_manager'
+require_relative 'fiber_job/client'
+require_relative 'fiber_job/cron_parser'
+require_relative 'fiber_job/cron'
+require_relative 'fiber_job/cron_job'
+
+# FiberJob is a high-performance, Redis-based background job processing library for Ruby
+# built on modern fiber-based concurrency. It combines the persistence of Redis with the
+# speed of async fibers to deliver exceptional performance and reliability.
+#
+# @example Basic usage
+#   # Configure the library
+#   FiberJob.configure do |config|
+#     config.redis_url = 'redis://localhost:6379/0'
+#     config.queues = { default: 5, high: 10 }
+#   end
+#
+#   # Define a job
+#   class EmailJob < FiberJob::Job
+#     def perform(user_id, message)
+#       # Send email logic here
+#     end
+#   end
+#
+#   # Enqueue a job
+#   EmailJob.perform_async(123, "Welcome!")
+#
+# @example Starting a worker
+#   worker = FiberJob::Worker.new(queues: ['default', 'high'])
+#   worker.start
+#
+module FiberJob
+  class << self
+    # Configures the FiberJob library with custom settings.
+    # Yields the configuration object to the provided block for customization.
+    #
+    # @example Basic configuration
+    #   FiberJob.configure do |config|
+    #     config.redis_url = 'redis://localhost:6379/0'
+    #     config.queues = { default: 5, high: 10, low: 2 }
+    #     config.log_level = :info
+    #   end
+    #
+    # @yield [config] Configuration object for customization
+    # @yieldparam config [FiberJob::Config] The configuration instance
+    # @return [void]
+    # @see FiberJob::Config
+    def configure
+      yield(config)
+    end
+
+    # Returns the current configuration instance.
+    # Creates a new configuration if one doesn't exist.
+    #
+    # @example Accessing configuration
+    #   redis_url = FiberJob.config.redis_url
+    #   queues = FiberJob.config.queues
+    #
+    # @return [FiberJob::Config] The current configuration instance
+    # @see FiberJob::Config
+    def config
+      @config ||= Config.new
+    end
+
+    # Returns the current logger instance.
+    # Creates a new logger with the current configuration if one doesn't exist.
+    #
+    # @example Logging messages
+    #   FiberJob.logger.info("Job completed successfully")
+    #   FiberJob.logger.error("Job failed: #{error.message}")
+    #
+    # @return [FiberJob::Logger] The current logger instance
+    # @see FiberJob::Logger
+    def logger
+      @logger ||= FiberJob::Logger.new(config)
+    end
+
+    # Automatically discovers and registers all cron job classes.
+    # Scans ObjectSpace for classes that inherit from FiberJob::CronJob
+    # and registers them for scheduling.
+    #
+    # @example Manual registration
+    #   class DailySummaryJob < FiberJob::CronJob
+    #     cron '0 9 * * *'  # Every day at 9 AM
+    #   end
+    #
+    #   FiberJob.register_cron_jobs
+    #
+    # @return [void]
+    # @see FiberJob::CronJob
+    # @see FiberJob::Cron
+    def register_cron_jobs
+      ObjectSpace.each_object(Class).select { |klass| klass < FiberJob::CronJob }.each(&:register)
+    end
+  end
+end
+
+# Register cron jobs after module is fully loaded
+FiberJob.register_cron_jobs

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: fiber_job
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Caio Mendonca
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: async
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.26.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.26.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.4.1
+executables:
+- fiber_job
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- README.md
+- bin/fiber_job
+- lib/fiber_job.rb
+- lib/fiber_job/client.rb
+- lib/fiber_job/concurrency.rb
+- lib/fiber_job/config.rb
+- lib/fiber_job/cron.rb
+- lib/fiber_job/cron_job.rb
+- lib/fiber_job/cron_parser.rb
+- lib/fiber_job/job.rb
+- lib/fiber_job/logger.rb
+- lib/fiber_job/process_manager.rb
+- lib/fiber_job/queue.rb
+- lib/fiber_job/version.rb
+- lib/fiber_job/worker.rb
+homepage: https://github.com/caieras/fiber_job
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/caieras/fiber_job
+  changelog_uri: https://github.com/caieras/fiber_job/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Experimental High-performance, Redis-based background job processing library
+  for Ruby built on fiber-based concurrency
+test_files: []