fiber_job 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: abd9dec492a67e4dc7c0e935e51fde3eaeae0d328555bba8cd35f617c83d65f1
-  data.tar.gz: af960358e61a9a743afaf14d23a909392cc942fdd76e3070e6ef8ae63a828255
+  metadata.gz: 2ea56b61adb2112a57ed3209847c220a7dd034669c74422df5a3ca3bf8f72e7e
+  data.tar.gz: 6d3f2e7c99f09432f82db73b1e3753d37ca7f1abd83034a578e0b8c5b2256da9
 SHA512:
-  metadata.gz: 06c83f8bd53d0d9a4319caffdc21336557eb976633a6d81140d0ed2c1f235c6bc396e1e6ee643666ea933bb5ef673d167ef769d35a42d8d0d752cd779a0b975d
-  data.tar.gz: 6e03a7dab3a23f19b78716f0aad0077d355b012458fbaa9a661ce2f90622f296c710f07dbf9d2b29c8970660e05980e3dc3c80aa9a7f342fff449c2290cdf99f
+  metadata.gz: 9320825973f4680e01992113a2494657bc858f173c430a4a5972c4661a0d1f9d2c8f5be34404cc93e98eec71ff8d2d7f8ffe06c61e624e3c7183246e667bc2c9
+  data.tar.gz: 3ac1833e277bd4875574c73575acf9745df976edd5804fabd30251b35d25c10ef5a77b282fdf3273afa17c210e50900bb4879df70bae82866f00c0c5baeb328c

data/lib/fiber_job/client.rb

@@ -39,8 +39,6 @@ module FiberJob
 
       queue_name = job_class.queue
       Queue.push(queue_name, payload)
-
-      FiberJob.logger.info "Enqueued #{job_class.name} with args: #{args.inspect}"
     end
 
     # Enqueues a job for execution after a specified delay.

data/lib/fiber_job/config.rb

@@ -3,6 +3,24 @@
 require 'logger'
 
 module FiberJob
+  # Null logger implementation for maximum performance when logging is disabled.
+  # All logging methods are no-ops, eliminating the overhead of log message formatting.
+  class NullLogger
+    %i[debug info warn error fatal unknown].each do |level|
+      define_method(level) { |*_args| nil }
+      define_method("#{level}?") { false }
+    end
+
+    def level=(*_args); end
+    def level = 0
+    def add(*_args); end
+    def log(*_args); end
+    def close; end
+    def reopen(*_args); end
+    def formatter=(*_args); end
+    def datetime_format=(*_args); end
+  end
+
   # Configuration class for FiberJob library settings.
   # Manages Redis connection, worker concurrency, queue configuration,
   # and logging settings. Supports both global and per-queue configuration.
@@ -44,7 +62,12 @@ module FiberJob
     #   @return [Symbol] Logging level (:debug, :info, :warn, :error)
     # @!attribute [rw] job_paths
     #   @return [Array<String>] List of paths to auto-load job classes from
-    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :logger, :log_level, :job_paths
+    # @!attribute [rw] pool_size
+    #   @return [Integer] Redis connection pool size
+    # @!attribute [rw] enable_logging
+    #   @return [Boolean] Whether to enable logging (set to false for maximum performance)
+    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :log_level, :job_paths, :pool_size,
+                  :enable_logging
 
     # Initializes configuration with sensible defaults.
     # Values can be overridden through environment variables or configuration blocks.
@@ -54,15 +77,19 @@ module FiberJob
     # Environment variables:
     # - REDIS_URL: Redis connection URL (default: redis://localhost:6379)
     # - FIBER_JOB_LOG_LEVEL: Logging level (default: info)
+    # - FIBER_JOB_CONCURRENCY: Global concurrency level (default: 2)
+    # - FIBER_JOB_ENABLE_LOGGING: Enable/disable logging (default: true)
+    # - FIBER_JOB_POOL_SIZE: Redis connection pool size (default: 5)
     def initialize
-      @redis_url = ENV['REDIS_URL'] || 'redis://localhost:6379'
-      @concurrency = 2
+      @redis_url = ENV.fetch('REDIS_URL', 'redis://localhost:6379')
+      @concurrency = ENV.fetch('FIBER_JOB_CONCURRENCY', 2).to_i
       @queues = [:default]
-      @queue_concurrency = { default: 2 }  # Per-queue concurrency
-      @log_level = ENV['FIBER_JOB_LOG_LEVEL']&.to_sym || :info
-      @logger = ::Logger.new($stdout)
-      @logger.level = ::Logger.const_get(@log_level.to_s.upcase)
+      @queue_concurrency = { default: 10 } # Per-queue concurrency
+      @log_level = ENV.fetch('FIBER_JOB_LOG_LEVEL', 'info').to_sym
+      @enable_logging = ENV.fetch('FIBER_JOB_ENABLE_LOGGING', 'true') == 'true'
+      @logger = nil # Lazy loaded
       @job_paths = []
+      @pool_size = ENV.fetch('FIBER_JOB_POOL_SIZE', 5).to_i
     end
 
     # Returns the concurrency setting for a specific queue.
@@ -100,24 +127,21 @@ module FiberJob
 
         @logger.info "Loading jobs from: #{path}"
 
-        Dir.glob("#{path}/**/*.rb").sort.each do |file|
-          begin
-            # Track classes before requiring the file
-            classes_before = job_classes
+        Dir.glob("#{path}/**/*.rb").each do |file|
+          # Track classes before requiring the file
+          classes_before = job_classes
 
-            require_relative File.expand_path(file)
+          require_relative File.expand_path(file)
 
-            # Find newly loaded job classes
-            new_classes = job_classes - classes_before
+          # Find newly loaded job classes
+          new_classes = job_classes - classes_before
 
-            new_classes.each do |job_class|
-              @logger.debug "Loaded job class: #{job_class}"
-              loaded_classes << job_class
-            end
-
-          rescue => e
-            @logger.error "Failed to load job file #{file}: #{e.message}"
+          new_classes.each do |job_class|
+            @logger.debug "Loaded job class: #{job_class}"
+            loaded_classes << job_class
           end
+        rescue StandardError => e
+          @logger.error "Failed to load job file #{file}: #{e.message}"
         end
       end
 
@@ -125,6 +149,29 @@ module FiberJob
       loaded_classes
     end
 
+    # Returns the logger instance, creating it lazily if needed.
+    # When logging is disabled, returns a null logger for maximum performance.
+    #
+    # @return [Logger, NullLogger] Logger instance or null logger
+    def logger
+      return @logger if @logger
+
+      @logger = if @enable_logging
+                  create_logger
+                else
+                  NullLogger.new
+                end
+    end
+
+    # Sets a custom logger instance.
+    # Setting a logger will override the enable_logging setting.
+    #
+    # @param value [Logger, nil] Logger instance or nil
+    def logger=(value)
+      @logger = value
+      @enable_logging = !value.nil?
+    end
+
     private
 
     # Returns all classes that inherit from FiberJob::Job
@@ -135,5 +182,12 @@ module FiberJob
         klass < FiberJob::Job
       end
     end
+
+    # Creates a logger instance with the configured level
+    def create_logger
+      logger = ::Logger.new($stdout)
+      logger.level = ::Logger.const_get(@log_level.to_s.upcase)
+      logger
+    end
   end
 end

data/lib/fiber_job/queue.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'redis'
+require 'connection_pool'
 
 module FiberJob
   # Queue provides Redis-based queue operations for job storage and retrieval.
@@ -35,7 +36,19 @@ module FiberJob
       end
     end
 
+    # Returns a Redis connection from the pool.
+    # This is the preferred method for getting Redis connections as it
+    # provides connection pooling and thread/fiber safety.
+    #
+    # @return [ConnectionPool] The Redis connection pool
+    def self.redis_pool
+      @redis_pool ||= ConnectionPool.new(size: config.pool_size, timeout: 5) do
+        Redis.new(url: config.redis_url)
+      end
+    end
+
     # Returns the shared Redis connection instance.
+    # @deprecated Use redis_pool.with { |redis| ... } instead for better performance
     # Creates a new connection if one doesn't exist.
     #
     # @return [Redis] The shared Redis connection
@@ -44,6 +57,7 @@ module FiberJob
     end
 
     # Creates a new Redis connection for fiber-safe operations.
+    # @deprecated Use redis_pool.with { |redis| ... } instead for better performance
     # Used when concurrent operations need separate connections.
     #
     # @return [Redis] A new Redis connection instance
@@ -64,7 +78,7 @@ module FiberJob
     #   payload = { 'class' => 'EmailJob', 'args' => [123, 'message'] }
     #   FiberJob::Queue.push(:default, payload)
     def self.push(queue_name, payload)
-      redis.lpush("queue:#{queue_name}", JSON.dump(payload))
+      redis_pool.with { |redis| redis.lpush("queue:#{queue_name}", JSON.dump(payload)) }
     end
 
     # Adds a job to the head of the queue for priority processing.
@@ -78,15 +92,16 @@ module FiberJob
     #   FiberJob::Queue.push_priority(:default, urgent_job_data)
     def self.push_priority(queue_name, payload)
       # Add to the head of the queue for priority execution
-      redis.rpush("queue:#{queue_name}", JSON.dump(payload))
+      redis_pool.with { |redis| redis.rpush("queue:#{queue_name}", JSON.dump(payload)) }
     end
 
     # Removes and returns a job from the specified queue.
     # Blocks for the specified timeout waiting for jobs to become available.
+    # Uses connection pool by default for better performance.
     #
     # @param queue_name [String, Symbol] Name of the source queue
     # @param timeout [Float] Maximum time to wait for a job (default: 0.1)
-    # @param redis_conn [Redis, nil] Optional Redis connection to use
+    # @param redis_conn [Redis, nil] Optional Redis connection to use (bypasses pool)
     # @return [Hash, nil] Job data hash or nil if timeout reached
     #
     # @example Pop from queue with timeout
@@ -95,8 +110,13 @@ module FiberJob
     #     puts "Processing job: #{job['class']}"
     #   end
     def self.pop(queue_name, timeout: 0.1, redis_conn: nil)
-      conn = redis_conn || redis
-      data = conn.brpop("queue:#{queue_name}", timeout: timeout)
+      data = if redis_conn
+               # Use provided connection (for legacy compatibility)
+               redis_conn.brpop("queue:#{queue_name}", timeout: timeout)
+             else
+               # Use connection pool for better performance
+               redis_pool.with { |redis| redis.brpop("queue:#{queue_name}", timeout: timeout) }
+             end
       data ? JSON.parse(data[1]) : nil
     end
 
@@ -113,7 +133,7 @@ module FiberJob
     #   future_time = Time.now.to_f + 3600  # 1 hour from now
     #   FiberJob::Queue.schedule(:default, job_data, future_time)
     def self.schedule(queue_name, payload, scheduled_at)
-      redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(payload))
+      redis_pool.with { |redis| redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(payload)) }
     end
 
     # Schedules a job with priority for execution at a specific time.
@@ -126,38 +146,49 @@ module FiberJob
     def self.schedule_priority(queue_name, payload, scheduled_at)
       # Mark as priority retry for head-of-queue execution
       priority_payload = payload.merge('priority_retry' => true)
-      redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(priority_payload))
+      redis_pool.with { |redis| redis.zadd("schedule:#{queue_name}", scheduled_at, JSON.dump(priority_payload)) }
     end
 
     # Processes scheduled jobs that are ready for execution.
     # Moves jobs from the scheduled set to the appropriate queue
     # when their scheduled time has arrived.
+    # Uses Redis pipeline for better performance when processing multiple jobs.
     #
     # @param queue_name [String, Symbol] Name of the queue to process
-    # @return [void]
+    # @return [Integer] Number of jobs moved to queue
     #
     # @example Process scheduled jobs
-    #   FiberJob::Queue.scheduled_jobs(:default)  # Called by workers
+    #   moved_count = FiberJob::Queue.scheduled_jobs(:default)  # Called by workers
     def self.scheduled_jobs(queue_name)
-      now = Time.now.to_f
-      jobs = redis.zrangebyscore("schedule:#{queue_name}", 0, now)
-
-      jobs.each do |job_json|
-        redis.zrem("schedule:#{queue_name}", job_json)
-        job_data = JSON.parse(job_json)
-
-        # Use priority queue for retries
-        if job_data['priority_retry']
-          job_data.delete('priority_retry') # Clean up the flag
-          push_priority(queue_name, job_data)
-        else
-          push(queue_name, job_data)
+      redis_pool.with do |redis|
+        now = Time.now.to_f
+        jobs = redis.zrangebyscore("schedule:#{queue_name}", 0, now)
+
+        return 0 if jobs.empty?
+
+        # Use pipeline for better performance with multiple operations
+        redis.pipelined do |pipeline|
+          jobs.each do |job_json|
+            pipeline.zrem("schedule:#{queue_name}", job_json)
+            job_data = JSON.parse(job_json)
+
+            # Use priority queue for retries
+            if job_data['priority_retry']
+              job_data.delete('priority_retry') # Clean up the flag
+              pipeline.rpush("queue:#{queue_name}", JSON.dump(job_data))
+            else
+              pipeline.lpush("queue:#{queue_name}", JSON.dump(job_data))
+            end
+          end
         end
+
+        jobs.size
       end
     end
 
     # Returns statistics for the specified queue.
     # Provides insight into queue depth, scheduled jobs, and processing status.
+    # Uses pipeline for efficient batch operations.
     #
     # @param queue_name [String, Symbol] Name of the queue
     # @return [Hash] Statistics hash with :size, :scheduled, and :processing keys
@@ -166,11 +197,19 @@ module FiberJob
     #   stats = FiberJob::Queue.stats(:default)
     #   puts "Pending: #{stats[:size]}, Scheduled: #{stats[:scheduled]}"
     def self.stats(queue_name)
-      {
-        size: redis.llen("queue:#{queue_name}"),
-        scheduled: redis.zcard("schedule:#{queue_name}"),
-        processing: redis.get("processing:#{queue_name}").to_i
-      }
+      redis_pool.with do |redis|
+        results = redis.pipelined do |pipeline|
+          pipeline.llen("queue:#{queue_name}")
+          pipeline.zcard("schedule:#{queue_name}")
+          pipeline.get("processing:#{queue_name}")
+        end
+
+        {
+          size: results[0],
+          scheduled: results[1],
+          processing: results[2].to_i
+        }
+      end
     end
 
     # Stores a failed job with error information for later analysis.
@@ -188,11 +227,11 @@ module FiberJob
     #   end
     def self.store_failed_job(job_data, error)
       failed_job_data = job_data.merge({
-        'failed_at' => Time.now.to_f,
-        'error' => error.message,
-        'backtrace' => error.backtrace&.first(10)
-      })
-      redis.lpush("failed", JSON.dump(failed_job_data))
+                                         'failed_at' => Time.now.to_f,
+                                         'error' => error.message,
+                                         'backtrace' => error.backtrace&.first(10)
+                                       })
+      redis_pool.with { |redis| redis.lpush('failed', JSON.dump(failed_job_data)) }
     end
 
     # Retrieves all failed jobs for inspection and debugging.
@@ -203,7 +242,9 @@ module FiberJob
     #   failed = FiberJob::Queue.failed_jobs
     #   failed.each { |job| puts "Failed: #{job['class']} - #{job['error']}" }
     def self.failed_jobs
-      redis.lrange("failed", 0, -1).map { |job_json| JSON.parse(job_json) }
+      redis_pool.with do |redis|
+        redis.lrange('failed', 0, -1).map { |job_json| JSON.parse(job_json) }
+      end
     end
   end
 end

data/lib/fiber_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberJob
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_job
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Caio Mendonca
@@ -37,6 +37,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 5.4.1
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
 executables:
 - fiber_job
 extensions: []