fiber_job 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a6c46d9a9278acfb132e182abe3f176e96e1b948aeaee30a722793b4b47b222
-  data.tar.gz: 70d45d9ee91164f8a5016bfc5cbae46dd4250de0a45d6658f442f4c4c78a72e2
+  metadata.gz: abd9dec492a67e4dc7c0e935e51fde3eaeae0d328555bba8cd35f617c83d65f1
+  data.tar.gz: af960358e61a9a743afaf14d23a909392cc942fdd76e3070e6ef8ae63a828255
 SHA512:
-  metadata.gz: f2a203f5d2124903dff417ca2562862a3326e496037e8d56e542629cbabbbfe1a129c9888bc31b047d522617db2afb18de22ec112d2c0ec9b37525f48046aedf
-  data.tar.gz: fc605f346003929db83d530b44024cc2f402c503041d1bb8ef890281c5762c581b943f628b289f4bcd5437a5391607c73db0a80c76391ff1de13d7cc5609220f
+  metadata.gz: 06c83f8bd53d0d9a4319caffdc21336557eb976633a6d81140d0ed2c1f235c6bc396e1e6ee643666ea933bb5ef673d167ef769d35a42d8d0d752cd779a0b975d
+  data.tar.gz: 6e03a7dab3a23f19b78716f0aad0077d355b012458fbaa9a661ce2f90622f296c710f07dbf9d2b29c8970660e05980e3dc3c80aa9a7f342fff449c2290cdf99f

data/README.md

@@ -2,24 +2,10 @@
 
 A high-performance, Redis-based background job processing library for Ruby built on modern fiber-based concurrency. FiberJob combines the persistence of Redis with the speed of async fibers to deliver exceptional performance and reliability.
 
-## Architecture Highlights
-
-FiberJob is a experimental gem that uses a architecture that sets it apart from traditional job queues:
-
-### **Hybrid Redis + Async::Queue Design**
-- **Redis for persistence**: Durable job storage with atomic operations and scheduling
-- **Async::Queue for speed**: Lightning-fast in-memory job processing with fiber-based concurrency
-- **Best of both worlds**: Reliability of Redis + performance of in-memory queues
-
-### **Advanced Fiber Management**
-- **Separation of concerns**: Independent polling fibers fetch from Redis while processing fibers execute jobs
-- **Per-queue fiber pools**: Isolated concurrency control with `Async::Semaphore` for optimal resource utilization
-- **Non-blocking operations**: All I/O operations use async/await patterns for maximum throughput
+## Requirements
 
-### **Production-Optimized Performance**
-- **Minimal Redis contention**: Single polling fiber per queue reduces Redis load
-- **Fast job execution**: Jobs flow through in-memory `Async::Queue` for sub-millisecond processing
-- **Scalable concurrency**: Configurable fiber pools scale efficiently without thread overhead
+- Ruby 3.1+
+- Redis 5.0+
 
 ## Features
 
@@ -266,11 +252,6 @@ end
 - `REDIS_URL`: Redis connection URL
 - `FIBER_JOB_LOG_LEVEL`: Logging level
 
-## Requirements
-
-- Ruby 3.1+
-- Redis 5.0+
-
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

data/bin/fiber_job

@@ -2,29 +2,71 @@
 # frozen_string_literal: true
 
 require_relative '../lib/fiber_job'
+require 'optparse'
 
-# Simple CLI for FiberJob
-command = ARGV[0]
+# CLI for FiberJob with configuration file support
+def load_configuration_file(config_path)
+  unless File.exist?(config_path)
+    puts "Error: Configuration file not found: #{config_path}"
+    exit 1
+  end
 
-case command
-when 'worker'
-  puts "Starting FiberJob worker..."
-  FiberJob::ProcessManager.start_worker
-when 'version'
-  puts "FiberJob version #{FiberJob::VERSION}"
-when nil, 'help'
+  begin
+    load config_path
+  rescue => e
+    puts "Error loading configuration file: #{e.message}"
+    exit 1
+  end
+end
+
+def show_help
   puts <<~HELP
     FiberJob - High-performance fiber-based job processing
 
     Usage:
-      fiber_job worker                Start a worker process
+      fiber_job start [options]       Start the worker server
       fiber_job version               Show version
       fiber_job help                  Show this help
 
+    Options:
+      -c, --config FILE               Path to configuration file
+
     Examples:
-      fiber_job worker                # Start worker with default configuration
-      REDIS_URL=redis://localhost:6379/1 fiber_job worker  # Custom Redis URL
+      fiber_job start                 # Start with default configuration
+      fiber_job start -c config.rb    # Start with custom configuration
+      REDIS_URL=redis://localhost:6379/1 fiber_job start  # Environment override
   HELP
+end
+
+# Parse command and options
+command = ARGV[0]
+config_file = nil
+
+case command
+when 'start'
+  # Parse options for start command
+  OptionParser.new do |opts|
+    opts.on('-c', '--config FILE', 'Configuration file path') do |file|
+      config_file = file
+    end
+    opts.on('-h', '--help', 'Show help') do
+      show_help
+      exit
+    end
+  end.parse!(ARGV[1..-1])
+
+  # Load configuration file if specified
+  load_configuration_file(config_file) if config_file
+
+  puts "Starting FiberJob server..."
+  FiberJob::ProcessManager.start_worker
+
+when 'version'
+  puts "FiberJob version #{FiberJob::VERSION}"
+
+when nil, 'help', '-h', '--help'
+  show_help
+
 else
   puts "Unknown command: #{command}"
   puts "Run 'fiber_job help' for usage information"

data/lib/fiber_job/concurrency.rb

@@ -4,15 +4,56 @@ require 'async'
 require 'async/semaphore'
 
 module FiberJob
-  # The ConcurrencyManager class is a lightweight wrapper around Async::Semaphore that
-  # controls how many jobs can execute simultaneously within each queue
+  # The ConcurrencyManager class provides efficient fiber reuse through direct
+  # semaphore acquire/release operations instead of spawning new fibers
   class ConcurrencyManager
+    attr_reader :semaphore
+
     def initialize(max_concurrency: 5)
       @semaphore = Async::Semaphore.new(max_concurrency)
     end
 
-    def execute(&block)
-      @semaphore.async(&block)
+    # Acquire a semaphore permit, blocking if none available
+    # @return [void]
+    def acquire
+      @semaphore.acquire
+    end
+
+    # Release a semaphore permit, allowing waiting fibers to proceed
+    # @return [void]
+    def release
+      @semaphore.release
+    end
+
+    # Execute a block with automatic acquire/release handling
+    # This method reuses the current fiber instead of spawning new ones
+    # @yield [void] Block to execute within semaphore protection
+    # @return [Object] Result of the block execution
+    def execute(&)
+      acquire
+      begin
+        yield
+      ensure
+        release
+      end
+    end
+
+    # Check if the semaphore would block on acquire
+    # @return [Boolean] true if acquire would block
+    def blocking?
+      @semaphore.blocking?
+    end
+
+    # Current number of acquired permits
+    # @return [Integer] Number of active permits
+    def count
+      @semaphore.count
+    end
+
+    # Maximum number of permits
+    # @return [Integer] Semaphore limit
+    def limit
+      @semaphore.limit
     end
   end
 end

data/lib/fiber_job/config.rb

@@ -24,6 +24,11 @@ module FiberJob
   #     }
   #   end
   #
+  # @example Job auto-loading
+  #   FiberJob.configure do |config|
+  #     config.job_paths = ['app/jobs', 'lib/jobs']
+  #   end
+  #
   class Config
     # @!attribute [rw] redis_url
     #   @return [String] Redis connection URL
@@ -37,7 +42,9 @@ module FiberJob
     #   @return [Logger] Logger instance for application logging
     # @!attribute [rw] log_level
     #   @return [Symbol] Logging level (:debug, :info, :warn, :error)
-    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :logger, :log_level
+    # @!attribute [rw] job_paths
+    #   @return [Array<String>] List of paths to auto-load job classes from
+    attr_accessor :redis_url, :concurrency, :queues, :queue_concurrency, :logger, :log_level, :job_paths
 
     # Initializes configuration with sensible defaults.
     # Values can be overridden through environment variables or configuration blocks.
@@ -49,12 +56,13 @@ module FiberJob
     # - FIBER_JOB_LOG_LEVEL: Logging level (default: info)
     def initialize
       @redis_url = ENV['REDIS_URL'] || 'redis://localhost:6379'
-      @concurrency = 2  # Global default fallback
+      @concurrency = 2
       @queues = [:default]
       @queue_concurrency = { default: 2 }  # Per-queue concurrency
       @log_level = ENV['FIBER_JOB_LOG_LEVEL']&.to_sym || :info
       @logger = ::Logger.new($stdout)
       @logger.level = ::Logger.const_get(@log_level.to_s.upcase)
+      @job_paths = []
     end
 
     # Returns the concurrency setting for a specific queue.
@@ -70,5 +78,62 @@ module FiberJob
     def concurrency_for_queue(queue_name)
       @queue_concurrency[queue_name.to_sym] || @concurrency
     end
+
+    # Auto-loads job classes from configured paths.
+    # Recursively loads all .rb files in the specified directories
+    # and validates that they contain classes inheriting from FiberJob::Job.
+    #
+    # @return [Array<Class>] List of loaded job classes
+    #
+    # @example Auto-load jobs
+    #   config.job_paths = ['app/jobs', 'lib/jobs']
+    #   loaded_jobs = config.load_jobs!
+    #   # => [EmailJob, DataProcessingJob, ...]
+    def load_jobs!
+      loaded_classes = []
+
+      @job_paths.each do |path|
+        unless Dir.exist?(path)
+          @logger.warn "Job path does not exist: #{path}"
+          next
+        end
+
+        @logger.info "Loading jobs from: #{path}"
+
+        Dir.glob("#{path}/**/*.rb").sort.each do |file|
+          begin
+            # Track classes before requiring the file
+            classes_before = job_classes
+
+            require_relative File.expand_path(file)
+
+            # Find newly loaded job classes
+            new_classes = job_classes - classes_before
+
+            new_classes.each do |job_class|
+              @logger.debug "Loaded job class: #{job_class}"
+              loaded_classes << job_class
+            end
+
+          rescue => e
+            @logger.error "Failed to load job file #{file}: #{e.message}"
+          end
+        end
+      end
+
+      @logger.info "Loaded #{loaded_classes.size} job classes"
+      loaded_classes
+    end
+
+    private
+
+    # Returns all classes that inherit from FiberJob::Job
+    def job_classes
+      return [] unless defined?(FiberJob::Job)
+
+      ObjectSpace.each_object(Class).select do |klass|
+        klass < FiberJob::Job
+      end
+    end
   end
 end

data/lib/fiber_job/job.rb

@@ -43,13 +43,18 @@ module FiberJob
     #   @return [Integer] Job priority (higher numbers = higher priority)
     # @!attribute [rw] timeout
     #   @return [Integer] Maximum execution time in seconds before timeout
+    # @!attribute [r] config
+    #   @return [FiberJob::Config] Configuration object for this job instance
     attr_accessor :queue, :retry_count, :max_retries, :priority, :timeout
+    attr_reader :config
 
     # Initializes a new job instance with default configuration.
-    # Sets reasonable defaults for queue, retries, and timeout values.
+    # Uses centralized configuration for defaults instead of hardcoded values.
     #
+    # @param config [FiberJob::Config, nil] Configuration object to use for defaults
     # @return [void]
-    def initialize
+    def initialize(config: nil)
+      @config = config || FiberJob.config
       @queue = :default
       @retry_count = 0
       @max_retries = 3

data/lib/fiber_job/process_manager.rb

@@ -1,18 +1,94 @@
 # frozen_string_literal: true
 
 module FiberJob
-  # ProcessManager is responsible for managing the lifecycle of workers
-  # and ensuring they run with the correct configuration.
+  # ProcessManager is the single entry point for running FiberJob workers.
+  # It manages worker lifecycle, configuration, signal handling, and graceful shutdown.
+  # 
+  # This is the recommended way to start FiberJob workers and should be used
+  # instead of directly instantiating Worker objects.
+  #
+  # @example Start with default configuration
+  #   FiberJob::ProcessManager.start_worker
+  #
+  # @example Start with custom configuration
+  #   config = FiberJob::Config.new
+  #   config.concurrency = 10
+  #   FiberJob::ProcessManager.start_worker(config: config)
+  #
   class ProcessManager
-    def self.start_worker(queues: nil, concurrency: nil)
-      queues ||= FiberJob.config.queues
-      concurrency ||= FiberJob.config.concurrency
-      worker = Worker.new(queues: queues, concurrency: concurrency)
+    class << self
+      # Starts a FiberJob worker process with proper signal handling and lifecycle management.
+      # This is the main entry point for running FiberJob workers.
+      #
+      # @param config [FiberJob::Config, nil] Configuration object (defaults to FiberJob.config)
+      # @param queues [Array<Symbol>, nil] Queue names to process (defaults to config.queues)  
+      # @param concurrency [Integer, nil] Concurrency level (defaults to config.concurrency)
+      # @return [void]
+      #
+      # @example Basic usage
+      #   FiberJob::ProcessManager.start_worker
+      #
+      # @example With custom configuration
+      #   FiberJob::ProcessManager.start_worker(
+      #     config: my_config,
+      #     queues: [:high_priority, :default],
+      #     concurrency: 8
+      #   )
+      def start_worker(config: nil, queues: nil, concurrency: nil)
+        config ||= FiberJob.config
+        queues ||= config.queues
+        concurrency ||= config.concurrency
 
-      trap('INT') { worker.stop }
-      trap('TERM') { worker.stop }
+        # Auto-load job classes if paths are configured
+        config.load_jobs! if config.job_paths&.any?
+        
+        log_startup_info(config, queues, concurrency)
+        
+        worker = create_worker(config: config, queues: queues, concurrency: concurrency)
+        setup_signal_handlers(worker)
+        
+        begin
+          worker.start
+        rescue => e
+          config.logger.error "Failed to start FiberJob worker: #{e.message}"
+          config.logger.error e.backtrace.join("\n")
+          exit 1
+        end
+      end
 
-      worker.start
+      private
+
+      # Creates and configures a new worker instance
+      def create_worker(config:, queues:, concurrency:)
+        Worker.new(config: config, queues: queues, concurrency: concurrency)
+      end
+
+      # Sets up signal handlers for graceful shutdown
+      def setup_signal_handlers(worker)
+        %w[INT TERM].each do |signal|
+          trap(signal) do
+            worker.config.logger.info "Received #{signal}, shutting down gracefully..."
+            worker.stop
+          end
+        end
+      end
+
+      # Logs startup information for debugging and monitoring
+      def log_startup_info(config, queues, concurrency)
+        logger = config.logger
+        logger.info "Starting FiberJob worker server"
+        logger.info "Redis URL: #{config.redis_url}"
+        logger.info "Queues: #{queues.join(', ')}"
+        logger.info "Global concurrency: #{concurrency}"
+        
+        queues.each do |queue|
+          queue_concurrency = config.concurrency_for_queue(queue)
+          worker_fibers = [queue_concurrency, 10].min
+          logger.info "Queue '#{queue}' concurrency: #{queue_concurrency} (#{worker_fibers} worker fibers)"
+        end
+        
+        logger.info "Log level: #{config.log_level}"
+      end
     end
   end
 end

data/lib/fiber_job/queue.rb

@@ -27,12 +27,20 @@ module FiberJob
   #
   # @see FiberJob::Worker
   class Queue
+    class << self
+      attr_writer :config
+
+      def config
+        @config ||= FiberJob.config
+      end
+    end
+
     # Returns the shared Redis connection instance.
     # Creates a new connection if one doesn't exist.
     #
     # @return [Redis] The shared Redis connection
     def self.redis
-      @redis ||= Redis.new(url: FiberJob.config.redis_url)
+      @redis ||= Redis.new(url: config.redis_url)
     end
 
     # Creates a new Redis connection for fiber-safe operations.
@@ -41,7 +49,7 @@ module FiberJob
     # @return [Redis] A new Redis connection instance
     def self.redis_connection
       # Create a new Redis connection for fiber-safe operations
-      Redis.new(url: FiberJob.config.redis_url)
+      Redis.new(url: config.redis_url)
     end
 
     # Adds a job to the specified queue for immediate processing.

data/lib/fiber_job/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FiberJob
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/fiber_job/worker.rb

@@ -4,13 +4,26 @@ require 'async'
 require 'async/queue'
 
 module FiberJob
+  # Worker handles the actual job processing using fiber-based concurrency.
+  #
+  # @private
+  # This class is intended for internal use by ProcessManager and should not
+  # be instantiated directly. Use FiberJob::ProcessManager.start_worker instead.
+  #
+  # @see FiberJob::ProcessManager
   class Worker
-    def initialize(queues: nil, concurrency: nil)
-      @queues = queues || FiberJob.config.queues
-      @concurrency = concurrency || FiberJob.config.concurrency
+    attr_reader :config
+
+    def initialize(config: nil, queues: nil, concurrency: nil)
+      @config = config || FiberJob.config
+      @queues = queues || @config.queues
+      @concurrency = concurrency || @config.concurrency
       @running = false
       @managers = {}
       @job_queues = {} # In-memory Async::Queue per Redis queue
+
+      # Inject configuration into Queue class
+      Queue.config = @config
     end
 
     def start
@@ -20,20 +33,19 @@ module FiberJob
         # Initialize all queues first
         @queues.each do |queue_name|
           @job_queues[queue_name] = Async::Queue.new
-          queue_concurrency = FiberJob.config.concurrency_for_queue(queue_name)
+          queue_concurrency = @config.concurrency_for_queue(queue_name)
           @managers[queue_name] = ConcurrencyManager.new(max_concurrency: queue_concurrency)
         end
 
-        # Start independent pollers for each queue
         @queues.each do |queue_name|
           task.async do
             poll_redis_queue(queue_name)
           end
         end
 
-        # Start independent worker pools for each queue
+        # Start fixed worker fiber pools for efficient fiber reuse
         @queues.each do |queue_name|
-          queue_concurrency = FiberJob.config.concurrency_for_queue(queue_name)
+          queue_concurrency = @config.concurrency_for_queue(queue_name) || @concurrency
           queue_concurrency.times do
             task.async do
               process_job_queue(queue_name)
@@ -41,7 +53,6 @@ module FiberJob
           end
         end
 
-        # Global support fibers
         task.async do
           process_scheduled_jobs
         end
@@ -62,36 +73,31 @@ module FiberJob
     private
 
     # Single poller fiber that fetches jobs from Redis and distributes to workers
-    # This eliminates Redis brpop contention by having only one fiber per queue accessing Redis
     def poll_redis_queue(queue_name)
-      # Create dedicated Redis connection for this poller to avoid blocking other pollers
       redis_conn = Queue.redis_connection
       while @running
         begin
-          # Use longer timeout since we're the only poller - no contention
           job_data = Queue.pop(queue_name, timeout: 1.0, redis_conn: redis_conn)
 
           if job_data
-            # Push to in-memory queue for worker fibers to process
             @job_queues[queue_name].push(job_data)
           end
         rescue StandardError => e
           FiberJob.logger.error "Redis polling error for queue #{queue_name}: #{e.message}"
-          sleep(1) # Brief pause on error
+          sleep(1)
         end
       end
     end
 
     # Worker fibers process jobs from the fast in-memory queue
-    # Multiple workers can process concurrently without Redis contention
+    # Each fiber reuses itself by acquiring/releasing semaphore permits
     def process_job_queue(queue_name)
       while @running
         begin
-          # Fast in-memory dequeue operation
           job_data = @job_queues[queue_name].dequeue
 
           if job_data
-            # Use semaphore to control actual job execution concurrency
+            # Reuse current fiber with semaphore-controlled concurrency
             @managers[queue_name].execute do
               execute_job(job_data)
             end
@@ -111,10 +117,6 @@ module FiberJob
 
       job.retry_count = job_data['retry_count'] || 0
 
-      # if job.retry_count > 0
-      #   FiberJob.logger.info "Executing #{job_class} (retry #{job.retry_count}/#{job.max_retries})"
-      # end
-
       begin
         Timeout.timeout(job.timeout) do
           args = (job_data['args'] || []).dup

data/lib/fiber_job.rb

@@ -105,4 +105,4 @@ module FiberJob
 end
 
 # Register cron jobs after module is fully loaded
-FiberJob.register_cron_jobs
+FiberJob.register_cron_jobs

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fiber_job
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Caio Mendonca