fiber_job 0.2.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 902c8f2047330d092fdcadda50067f8bef6a47bc43039ed6bad7172d6976a1b7
-  data.tar.gz: 9cd001ba179f3f6a16619441b448e1d46dc7917431a3deea753615135af499f7
+  metadata.gz: e66c22f05345c01f395ff4c12d2552166dacdbf793beb21d33a80d5039d8614b
+  data.tar.gz: d05d0d697c0160b4b1265e10034cc8f6b305b6b6b71cb08394225f5d2d3573b4
 SHA512:
-  metadata.gz: 391d2e8826377526d5c852f1ba1a07a84767d3c57544d73ce93cb7cb6e3f3cf381591e6a76049219d3122259af047d6f3dce492a00aea764319f53583ba20385
-  data.tar.gz: e5dce3e93ae978c1c0d02169cfe2b95cd92888a92d9f35675f372cd31fe6ae17e2a6a551a977abaed40095e2d5de3068dc94cbe59b69c0a01ef54ed005b46ab8
+  metadata.gz: 71fd43833e9d47668af585d97cfe114950fe59e0197bbc2597e8b0b479c97c5d19eef8a877690f2f8910aaadccdc05718e938eed870e62e24a14a0d23a92d8dc
+  data.tar.gz: 7234510238408774dfba23fb6377d4b0a21be216ec2b549ff25bc9a27ecf29f0970b374fd12e82d4c992a60c67e203875e14530c1e46c263907513ba023f376c

data/lib/fiber_job/config.rb

@@ -3,6 +3,24 @@
 require 'logger'
 
 module FiberJob
+  # Null logger implementation for maximum performance when logging is disabled.
+  # All logging methods are no-ops, eliminating the overhead of log message formatting.
+  class NullLogger
+    %i[debug info warn error fatal unknown].each do |level|
+      define_method(level) { |*_args| nil }
+      define_method("#{level}?") { false }
+    end
+
+    def level=(*_args); end
+    def level = 0
+    def add(*_args); end
+    def log(*_args); end
+    def close; end
+    def reopen(*_args); end
+    def formatter=(*_args); end
+    def datetime_format=(*_args); end
+  end
+
   # Configuration class for FiberJob library settings.
   # Manages Redis connection, worker concurrency, queue configuration,
   # and logging settings. Supports both global and per-queue configuration.
@@ -44,7 +62,12 @@ module FiberJob
     #   @return [Symbol] Logging level (:debug, :info, :warn, :error)
     # @!attribute [rw] job_paths
     #   @return [Array<String>] List of paths to auto-load job classes from
-    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :logger, :log_level, :job_paths
+    # @!attribute [rw] pool_size
+    #   @return [Integer] Redis connection pool size
+    # @!attribute [rw] enable_logging
+    #   @return [Boolean] Whether to enable logging (set to false for maximum performance)
+    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :log_level, :job_paths, :pool_size,
+                  :enable_logging
 
     # Initializes configuration with sensible defaults.
     # Values can be overridden through environment variables or configuration blocks.
@@ -54,15 +77,19 @@ module FiberJob
     # Environment variables:
     # - REDIS_URL: Redis connection URL (default: redis://localhost:6379)
     # - FIBER_JOB_LOG_LEVEL: Logging level (default: info)
+    # - FIBER_JOB_CONCURRENCY: Global concurrency level (default: 2)
+    # - FIBER_JOB_ENABLE_LOGGING: Enable/disable logging (default: true)
+    # - FIBER_JOB_POOL_SIZE: Redis connection pool size (default: 5)
     def initialize
-      @redis_url = ENV['REDIS_URL'] || 'redis://localhost:6379'
-      @concurrency = 2
+      @redis_url = ENV.fetch('REDIS_URL', 'redis://localhost:6379')
+      @concurrency = ENV.fetch('FIBER_JOB_CONCURRENCY', 2).to_i
       @queues = [:default]
-      @queue_concurrency = { default: 2 }  # Per-queue concurrency
-      @log_level = ENV['FIBER_JOB_LOG_LEVEL']&.to_sym || :info
-      @logger = ::Logger.new($stdout)
-      @logger.level = ::Logger.const_get(@log_level.to_s.upcase)
+      @queue_concurrency = { default: 10 } # Per-queue concurrency
+      @log_level = ENV.fetch('FIBER_JOB_LOG_LEVEL', 'info').to_sym
+      @enable_logging = ENV.fetch('FIBER_JOB_ENABLE_LOGGING', 'true') == 'true'
+      @logger = nil # Lazy loaded
       @job_paths = []
+      @pool_size = ENV.fetch('FIBER_JOB_POOL_SIZE', 5).to_i
     end
 
     # Returns the concurrency setting for a specific queue.
@@ -98,33 +125,50 @@ module FiberJob
           next
         end
 
-        @logger.info "Loading jobs from: #{path}"
+        Dir.glob("#{path}/**/*.rb").each do |file|
+          # Track classes before requiring the file
+          classes_before = job_classes
 
-        Dir.glob("#{path}/**/*.rb").sort.each do |file|
-          begin
-            # Track classes before requiring the file
-            classes_before = job_classes
+          require_relative File.expand_path(file)
 
-            require_relative File.expand_path(file)
+          # Find newly loaded job classes
+          new_classes = job_classes - classes_before
 
-            # Find newly loaded job classes
-            new_classes = job_classes - classes_before
-
-            new_classes.each do |job_class|
-              @logger.debug "Loaded job class: #{job_class}"
-              loaded_classes << job_class
-            end
-
-          rescue => e
-            @logger.error "Failed to load job file #{file}: #{e.message}"
+          new_classes.each do |job_class|
+            @logger.debug "Loaded job class: #{job_class}"
+            loaded_classes << job_class
           end
+        rescue StandardError => e
+          @logger.error "Failed to load job file #{file}: #{e.message}"
         end
       end
 
-      @logger.info "Loaded #{loaded_classes.size} job classes"
       loaded_classes
     end
 
+    # Returns the logger instance, creating it lazily if needed.
+    # When logging is disabled, returns a null logger for maximum performance.
+    #
+    # @return [Logger, NullLogger] Logger instance or null logger
+    def logger
+      return @logger if @logger
+
+      @logger = if @enable_logging
+                  create_logger
+                else
+                  NullLogger.new
+                end
+    end
+
+    # Sets a custom logger instance.
+    # Setting a logger will override the enable_logging setting.
+    #
+    # @param value [Logger, nil] Logger instance or nil
+    def logger=(value)
+      @logger = value
+      @enable_logging = !value.nil?
+    end
+
     private
 
     # Returns all classes that inherit from FiberJob::Job
@@ -135,5 +179,12 @@ module FiberJob
         klass < FiberJob::Job
       end
     end
+
+    # Creates a logger instance with the configured level
+    def create_logger
+      logger = ::Logger.new($stdout)
+      logger.level = ::Logger.const_get(@log_level.to_s.upcase)
+      logger
+    end
   end
 end

data/lib/fiber_job/queue.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'redis'
+require 'connection_pool'
 
 module FiberJob
   # Queue provides Redis-based queue operations for job storage and retrieval.
@@ -35,7 +36,19 @@ module FiberJob
       end
     end
 
+    # Returns a Redis connection from the pool.
+    # This is the preferred method for getting Redis connections as it
+    # provides connection pooling and thread/fiber safety.
+    #
+    # @return [ConnectionPool] The Redis connection pool
+    def self.redis_pool
+      @redis_pool ||= ConnectionPool.new(size: config.pool_size, timeout: 5) do
+        Redis.new(url: config.redis_url)
+      end
+    end
+
     # Returns the shared Redis connection instance.
+    # @deprecated Use redis_pool.with { |redis| ... } instead for better performance
     # Creates a new connection if one doesn't exist.
     #
     # @return [Redis] The shared Redis connection
@@ -44,6 +57,7 @@ module FiberJob
     end
 
     # Creates a new Redis connection for fiber-safe operations.
+    # @deprecated Use redis_pool.with { |redis| ... } instead for better performance
     # Used when concurrent operations need separate connections.
     #
     # @return [Redis] A new Redis connection instance
@@ -64,7 +78,7 @@ module FiberJob
     #   payload = { 'class' => 'EmailJob', 'args' => [123, 'message'] }
     #   FiberJob::Queue.push(:default, payload)
     def self.push(queue_name, payload)
-      redis.lpush("queue:#{queue_name}", JSON.dump(payload))
+      redis_pool.with { |redis| redis.lpush("queue:#{queue_name}", JSON.dump(payload)) }
     end
 
     # Adds a job to the head of the queue for priority processing.
@@ -78,15 +92,16 @@ module FiberJob
     #   FiberJob::Queue.push_priority(:default, urgent_job_data)
     def self.push_priority(queue_name, payload)
       # Add to the head of the queue for priority execution
-      redis.rpush("queue:#{queue_name}", JSON.dump(payload))
+      redis_pool.with { |redis| redis.rpush("queue:#{queue_name}", JSON.dump(payload)) }
     end
 
     # Removes and returns a job from the specified queue.
     # Blocks for the specified timeout waiting for jobs to become available.
+    # Uses connection pool by default for better performance.
     #
     # @param queue_name [String, Symbol] Name of the source queue
     # @param timeout [Float] Maximum time to wait for a job (default: 0.1)
-    # @param redis_conn [Redis, nil] Optional Redis connection to use
+    # @param redis_conn [Redis, nil] Optional Redis connection to use (bypasses pool)
     # @return [Hash, nil] Job data hash or nil if timeout reached
     #
     # @example Pop from queue with timeout
@@ -95,8 +110,13 @@ module FiberJob
     #     puts "Processing job: #{job['class']}"
     #   end
     def self.pop(queue_name, timeout: 0.1, redis_conn: nil)
-      conn = redis_conn || redis
-      data = conn.brpop("queue:#{queue_name}", timeout: timeout)
+      data = if redis_conn
+               # Use provided connection (for legacy compatibility)
+               redis_conn.brpop("queue:#{queue_name}", timeout: timeout)
+             else
+               # Use connection pool for better performance
+               redis_pool.with { |redis| redis.brpop("queue:#{queue_name}", timeout: timeout) }
+             end
       data ? JSON.parse(data[1]) : nil
     end
 
@@ -113,7 +133,7 @@ module FiberJob
     #   future_time = Time.now.to_f + 3600  # 1 hour from now
     #   FiberJob::Queue.schedule(:default, job_data, future_time)
     def self.schedule(queue_name, payload, scheduled_at)
-      redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(payload))
+      redis_pool.with { |redis| redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(payload)) }
     end
 
     # Schedules a job with priority for execution at a specific time.
@@ -126,38 +146,49 @@ module FiberJob
     def self.schedule_priority(queue_name, payload, scheduled_at)
       # Mark as priority retry for head-of-queue execution
       priority_payload = payload.merge('priority_retry' => true)
-      redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(priority_payload))
+      redis_pool.with { |redis| redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(priority_payload)) }
     end
 
     # Processes scheduled jobs that are ready for execution.
     # Moves jobs from the scheduled set to the appropriate queue
     # when their scheduled time has arrived.
+    # Uses Redis pipeline for better performance when processing multiple jobs.
     #
     # @param queue_name [String, Symbol] Name of the queue to process
-    # @return [void]
+    # @return [Integer] Number of jobs moved to queue
     #
     # @example Process scheduled jobs
-    #   FiberJob::Queue.scheduled_jobs(:default)  # Called by workers
+    #   moved_count = FiberJob::Queue.scheduled_jobs(:default)  # Called by workers
     def self.scheduled_jobs(queue_name)
-      now = Time.now.to_f
-      jobs = redis.zrangebyscore("schedule:#{queue_name}", 0, now)
-
-      jobs.each do |job_json|
-        redis.zrem("schedule:#{queue_name}", job_json)
-        job_data = JSON.parse(job_json)
-
-        # Use priority queue for retries
-        if job_data['priority_retry']
-          job_data.delete('priority_retry') # Clean up the flag
-          push_priority(queue_name, job_data)
-        else
-          push(queue_name, job_data)
+      redis_pool.with do |redis|
+        now = Time.now.to_f
+        jobs = redis.zrangebyscore("schedule:#{queue_name}", 0, now)
+
+        return 0 if jobs.empty?
+
+        # Use pipeline for better performance with multiple operations
+        redis.pipelined do |pipeline|
+          jobs.each do |job_json|
+            pipeline.zrem("schedule:#{queue_name}", job_json)
+            job_data = JSON.parse(job_json)
+
+            # Use priority queue for retries
+            if job_data['priority_retry']
+              job_data.delete('priority_retry') # Clean up the flag
+              pipeline.rpush("queue:#{queue_name}", JSON.dump(job_data))
+            else
+              pipeline.lpush("queue:#{queue_name}", JSON.dump(job_data))
+            end
+          end
         end
+
+        jobs.size
       end
     end
 
     # Returns statistics for the specified queue.
     # Provides insight into queue depth, scheduled jobs, and processing status.
+    # Uses pipeline for efficient batch operations.
     #
     # @param queue_name [String, Symbol] Name of the queue
     # @return [Hash] Statistics hash with :size, :scheduled, and :processing keys
@@ -166,11 +197,19 @@ module FiberJob
     #   stats = FiberJob::Queue.stats(:default)
     #   puts "Pending: #{stats[:size]}, Scheduled: #{stats[:scheduled]}"
     def self.stats(queue_name)
-      {
-        size: redis.llen("queue:#{queue_name}"),
-        scheduled: redis.zcard("schedule:#{queue_name}"),
-        processing: redis.get("processing:#{queue_name}").to_i
-      }
+      redis_pool.with do |redis|
+        results = redis.pipelined do |pipeline|
+          pipeline.llen("queue:#{queue_name}")
+          pipeline.zcard("schedule:#{queue_name}")
+          pipeline.get("processing:#{queue_name}")
+        end
+
+        {
+          size: results[0],
+          scheduled: results[1],
+          processing: results[2].to_i
+        }
+      end
     end
 
     # Stores a failed job with error information for later analysis.
@@ -188,11 +227,11 @@ module FiberJob
     #   end
     def self.store_failed_job(job_data, error)
       failed_job_data = job_data.merge({
-        'failed_at' => Time.now.to_f,
-        'error' => error.message,
-        'backtrace' => error.backtrace&.first(10)
-      })
-      redis.lpush("failed", JSON.dump(failed_job_data))
+                                         'failed_at' => Time.now.to_f,
+                                         'error' => error.message,
+                                         'backtrace' => error.backtrace&.first(10)
+                                       })
+      redis_pool.with { |redis| redis.lpush('failed', JSON.dump(failed_job_data)) }
     end
 
     # Retrieves all failed jobs for inspection and debugging.
@@ -203,7 +242,9 @@ module FiberJob
     #   failed = FiberJob::Queue.failed_jobs
     #   failed.each { |job| puts "Failed: #{job['class']} - #{job['error']}" }
     def self.failed_jobs
-      redis.lrange("failed", 0, -1).map { |job_json| JSON.parse(job_json) }
+      redis_pool.with do |redis|
+        redis.lrange('failed', 0, -1).map { |job_json| JSON.parse(job_json) }
+      end
     end
   end
 end

data/lib/fiber_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberJob
-  VERSION = "0.2.1"
-end
+  VERSION = '0.2.3'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_job
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Caio Mendonca
@@ -37,6 +37,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 5.4.1
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
 executables:
 - fiber_job
 extensions: []