io-complyance-unify-sdk 3.0.0

data/lib/complyance/circuit_breaker.rb

@@ -0,0 +1,99 @@
+require 'logger'
+
+module Complyance
+  # Circuit Breaker implementation for handling service failures
+  class CircuitBreaker
+    STATE_CLOSED = 'CLOSED'
+    STATE_OPEN = 'OPEN'
+    STATE_HALF_OPEN = 'HALF_OPEN'
+
+    attr_reader :state, :failure_count, :last_failure_time
+
+    # Initialize circuit breaker
+    # @param config [Hash] Circuit breaker configuration
+    # @option config [Integer] :failure_threshold Number of failures before opening circuit
+    # @option config [Integer] :reset_timeout Timeout in seconds before attempting reset
+    def initialize(config = {})
+      @state = STATE_CLOSED
+      @failure_count = 0
+      @last_failure_time = 0
+      @failure_threshold = config[:failure_threshold] || 3
+      @reset_timeout = config[:reset_timeout] || 60
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+    end
+
+    # Execute a function with circuit breaker protection
+    # @param operation [Proc] Function to execute
+    # @return [Object] Result of operation
+    # @raise [RuntimeError] If circuit is open
+    def execute(&operation)
+      check_state
+      
+      begin
+        result = operation.call
+        record_success
+        result
+      rescue => e
+        record_failure
+        raise e
+      end
+    end
+
+    # Check if circuit breaker is currently open
+    # @return [Boolean] True if open
+    def open?
+      @state == STATE_OPEN
+    end
+
+    private
+
+    # Check if circuit breaker should allow operation
+    # @raise [RuntimeError] If circuit is open
+    def check_state
+      if @state == STATE_OPEN
+        remaining_time = get_remaining_timeout
+        if remaining_time > 0
+          @logger.debug "Circuit breaker is OPEN - #{remaining_time} seconds remaining"
+          raise "Circuit breaker is open - #{remaining_time} seconds remaining"
+        end
+
+        # Reset timeout expired, move to half-open
+        @state = STATE_HALF_OPEN
+        @logger.info "Circuit breaker state changed to HALF-OPEN"
+      end
+    end
+
+    # Record a successful operation
+    def record_success
+      if @state == STATE_HALF_OPEN
+        @state = STATE_CLOSED
+        @failure_count = 0
+        @last_failure_time = 0
+        @logger.info "Circuit breaker reset - state changed to CLOSED"
+      end
+    end
+
+    # Record a failed operation
+    def record_failure
+      @failure_count += 1
+      @last_failure_time = Time.now.to_i
+
+      if @state == STATE_HALF_OPEN || 
+         (@state == STATE_CLOSED && @failure_count >= @failure_threshold)
+        @state = STATE_OPEN
+        @logger.warn "Circuit breaker opened after #{@failure_count} failures"
+      end
+    end
+
+    # Get remaining time before circuit breaker resets
+    # @return [Integer] Seconds remaining, 0 if timeout expired
+    def get_remaining_timeout
+      return 0 if @last_failure_time == 0
+
+      elapsed = Time.now.to_i - @last_failure_time
+      remaining = @reset_timeout - elapsed
+      [0, remaining].max
+    end
+  end
+end

data/lib/complyance/persistent_queue_manager.rb

@@ -0,0 +1,474 @@
+require 'logger'
+require 'json'
+require 'fileutils'
+require 'monitor'
+require_relative 'circuit_breaker'
+require_relative 'gets_unify_sdk'
+
+module Complyance
+  # Manages persistent queue for document submissions
+  class PersistentQueueManager
+    QUEUE_DIR = 'complyance-queue'
+    PENDING_DIR = 'pending'
+    PROCESSING_DIR = 'processing'
+    FAILED_DIR = 'failed'
+    SUCCESS_DIR = 'success'
+
+    attr_reader :is_running
+
+    # Initialize queue manager
+    # @param api_key [String] API key for authentication
+    # @param local [Boolean] Whether to use local environment
+    # @param circuit_breaker [CircuitBreaker, nil] Optional shared circuit breaker
+    def initialize(api_key, local, circuit_breaker = nil)
+      @api_key = api_key
+      @local = local
+      @queue_base_path = queue_base_path
+      @logger = Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+      @processing_lock = Monitor.new
+      @is_running = false
+
+      # Initialize circuit breaker with 3 failure threshold and 1 minute timeout
+      @circuit_breaker = circuit_breaker || CircuitBreaker.new(
+        failure_threshold: 3,
+        reset_timeout: 60
+      )
+
+      initialize_queue_directories
+      @logger.info "PersistentQueueManager initialized with queue directory: #{@queue_base_path}"
+
+      # Automatically start processing and retry any existing failed submissions
+      start_processing
+      retry_failed_submissions
+    end
+
+    # Get base path for queue directories
+    # @return [String] Base path
+    def queue_base_path
+      File.join(ENV['HOME'] || ENV['USERPROFILE'], QUEUE_DIR)
+    end
+
+    # Initialize queue directory structure
+    def initialize_queue_directories
+      [PENDING_DIR, PROCESSING_DIR, FAILED_DIR, SUCCESS_DIR].each do |dir|
+        path = File.join(@queue_base_path, dir)
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      end
+      @logger.debug "Queue directories initialized"
+    end
+
+    # Enqueue a submission for processing
+    # @param submission [PayloadSubmission] Submission to enqueue
+    def enqueue(submission)
+      begin
+        file_name = generate_file_name(submission)
+        file_path = File.join(@queue_base_path, PENDING_DIR, file_name)
+
+        # Check if file already exists (same document ID)
+        if File.exist?(file_path)
+          @logger.info "Document already exists in queue: #{file_name}. Skipping duplicate submission."
+          return
+        end
+
+        # Parse the UnifyRequest JSON string
+        json_payload = submission.payload
+        @logger.debug "Queue: Received payload with length: #{json_payload.bytesize} characters"
+
+        # Verify the payload is not empty
+        if json_payload.strip.empty? || json_payload == '{}'
+          @logger.error "🔥 QUEUE: ERROR - Received empty or invalid payload: '#{json_payload}'"
+          raise "Cannot enqueue empty payload"
+        end
+
+        # Parse the UnifyRequest JSON string to a proper JSON object
+        unify_request_map = JSON.parse(json_payload)
+
+        # Create submission record
+        record = {
+          payload: unify_request_map,
+          source_id: "#{submission.source.name}:#{submission.source.version}",
+          country: submission.country.to_s,
+          document_type: submission.document_type.to_s,
+          enqueued_at: Time.now.iso8601,
+          timestamp: (Time.now.to_f * 1000).to_i # Convert to milliseconds for consistency
+        }
+
+        # Write to file
+        File.write(file_path, JSON.pretty_generate(record))
+
+        @logger.info "Enqueued submission: #{file_name} for source: #{record[:source_id]}, country: #{record[:country]}"
+
+        # Start processing if not already running
+        start_processing
+
+      rescue => e
+        @logger.error "Failed to enqueue submission to persistent storage: #{e.message}"
+        raise "Failed to persist submission: #{e.message}"
+      end
+    end
+
+    # Generate unique filename for submission
+    # @param submission [PayloadSubmission] Submission to generate filename for
+    # @return [String] Generated filename
+    def generate_file_name(submission)
+      # Extract document ID from payload
+      document_id = extract_document_id(submission.payload)
+
+      # Generate filename using source and document ID for unique reference
+      source_id = "#{submission.source.name}:#{submission.source.version}".gsub(/[^\w-]/, '_')
+      country = submission.country.to_s
+
+      format("%s_%s_%s_%s.json",
+        source_id,
+        document_id,
+        country,
+        submission.document_type.to_s
+      )
+    end
+
+    # Extract document ID from payload JSON
+    # @param payload [String] JSON payload
+    # @return [String] Extracted document ID
+    def extract_document_id(payload)
+      data = JSON.parse(payload)
+      if data.dig('payload', 'invoice_data', 'invoice_number')
+        data['payload']['invoice_data']['invoice_number']
+      else
+        "doc_#{Time.now.to_i}"
+      end
+    rescue => e
+      @logger.warn "Failed to extract document ID from payload: #{e.message}"
+      "doc_#{Time.now.to_i}"
+    end
+
+    # Start processing queue items
+    def start_processing
+      unless @is_running
+        @is_running = true
+        @logger.info "Started persistent queue processing"
+
+        # Start background processing in a new thread
+        @processing_thread = Thread.new do
+          while @is_running
+            process_pending_submissions
+            sleep(0.5) # 500ms delay
+          end
+        end
+      end
+    end
+
+    # Process pending submissions immediately
+    def process_pending_submissions_now
+      @logger.info "Manually triggering processing of pending submissions"
+
+      # Check circuit breaker state before manual processing
+      if @circuit_breaker.open?
+        current_time = Time.now.to_i
+        time_since_last_failure = current_time - @circuit_breaker.last_failure_time
+
+        if time_since_last_failure < 60 # 1 minute
+          remaining_time = 60 - time_since_last_failure
+          @logger.info "🚫 Circuit breaker is OPEN - remaining time: #{remaining_time}s. Manual processing skipped."
+          return
+        else
+          @logger.info "✅ Circuit breaker timeout expired (#{time_since_last_failure}s) - proceeding with manual processing"
+        end
+      else
+        @logger.info "✅ Circuit breaker is CLOSED - proceeding with manual processing"
+      end
+
+      process_pending_submissions
+    end
+
+    # Stop processing queue items
+    def stop_processing
+      @is_running = false
+      @processing_thread&.join
+      @logger.info "Stopped persistent queue processing"
+    end
+
+    # Process pending submissions in queue
+    def process_pending_submissions
+      return unless @is_running
+
+      @processing_lock.synchronize do
+        begin
+          pending_dir = File.join(@queue_base_path, PENDING_DIR)
+          files = Dir[File.join(pending_dir, '*.json')]
+
+          if files.empty?
+            @logger.debug "No pending submissions to process"
+            return
+          end
+
+          @logger.debug "Found #{files.size} pending submissions in queue"
+
+          # Check circuit breaker state before attempting to process
+          if @circuit_breaker.open?
+            current_time = Time.now.to_i
+            time_since_last_failure = current_time - @circuit_breaker.last_failure_time
+
+            if time_since_last_failure < 60 # 1 minute
+              remaining_time = 60 - time_since_last_failure
+              @logger.debug "Circuit breaker is OPEN - #{remaining_time} seconds remaining. Queue has #{files.size} items waiting."
+              return
+            else
+              @logger.debug "Circuit breaker timeout expired - attempting to process #{files.size} queued items"
+            end
+          end
+
+          files.each do |file_path|
+            if File.exist?(file_path)
+              begin
+                process_submission_file(file_path)
+              rescue => e
+                @logger.error "Failed to process queued submission #{file_path}: #{e.message}"
+              end
+            end
+          end
+
+        rescue => e
+          @logger.error "Error processing pending submissions: #{e.message}"
+        end
+      end
+    end
+
+    # Process a single submission file
+    # @param file_path [String] Path to submission file
+    def process_submission_file(file_path)
+      begin
+        # Read submission record
+        record = JSON.parse(File.read(file_path))
+        file_name = File.basename(file_path)
+        processing_path = File.join(@queue_base_path, PROCESSING_DIR, file_name)
+
+        # Move to processing directory
+        FileUtils.mv(file_path, processing_path)
+
+        @logger.debug "Processing submission: #{file_name} for source: #{record['source_id']}"
+
+        begin
+          # Convert stored payload back to UnifyRequest
+          unify_request = UnifyRequest.new(record['payload'])
+
+          # Use the SDK's pushToUnify method with circuit breaker
+          response = @circuit_breaker.execute do
+            GETSUnifySDK.push_to_unify(unify_request)
+          end
+
+          # Check for success
+          is_success = false
+          if response
+            is_success = response.success?
+
+            # Check submission status
+            if response.data&.submission
+              submission = response.data.submission
+              is_success ||= submission.accepted? || 
+                           submission.status.to_s.downcase == 'accepted'
+            end
+
+            # Check document status
+            if response.data&.document
+              document = response.data.document
+              is_success ||= document.status.to_s.downcase == 'success'
+            end
+          end
+
+          if is_success
+            @logger.info "Queue: SUCCESS - Removing file from queue: #{file_name}"
+            File.delete(processing_path)
+            return
+          else
+            status = response ? response.status : 'null'
+            @logger.warn "Queue: NON-SUCCESS - Moving to failed directory. Status: '#{status}'"
+            raise "API returned non-success status: #{status}"
+          end
+
+        rescue => e
+          @logger.error "Failed to send queued submission via pushToUnify: #{e.message}"
+
+          failed_path = File.join(@queue_base_path, FAILED_DIR, file_name)
+
+          if File.exist?(failed_path)
+            File.delete(processing_path)
+          else
+            FileUtils.mv(processing_path, failed_path)
+          end
+
+          raise e
+        end
+
+      rescue => e
+        @logger.warn "Failed to process submission: #{File.basename(file_path)} - Error: #{e.message}"
+
+        file_name = File.basename(file_path)
+        processing_path = File.join(@queue_base_path, PROCESSING_DIR, file_name)
+        failed_path = File.join(@queue_base_path, FAILED_DIR, file_name)
+
+        # Move to failed directory
+        if File.exist?(processing_path)
+          if File.exist?(failed_path)
+            File.delete(processing_path)
+          else
+            FileUtils.mv(processing_path, failed_path)
+          end
+        elsif File.exist?(file_path)
+          if File.exist?(failed_path)
+            File.delete(file_path)
+          else
+            FileUtils.mv(file_path, failed_path)
+          end
+        end
+      end
+    end
+
+    # Get current queue status
+    # @return [Hash] Queue status information
+    def queue_status
+      begin
+        pending_count = Dir[File.join(@queue_base_path, PENDING_DIR, '*.json')].size
+        processing_count = Dir[File.join(@queue_base_path, PROCESSING_DIR, '*.json')].size
+        failed_count = Dir[File.join(@queue_base_path, FAILED_DIR, '*.json')].size
+        success_count = Dir[File.join(@queue_base_path, SUCCESS_DIR, '*.json')].size
+
+        {
+          pending_count: pending_count,
+          processing_count: processing_count,
+          failed_count: failed_count,
+          success_count: success_count,
+          is_running: @is_running
+        }
+
+      rescue => e
+        @logger.error "Failed to get queue status: #{e.message}"
+        {
+          pending_count: 0,
+          processing_count: 0,
+          failed_count: 0,
+          success_count: 0,
+          is_running: false
+        }
+      end
+    end
+
+    # Retry failed submissions
+    def retry_failed_submissions
+      begin
+        failed_dir = File.join(@queue_base_path, FAILED_DIR)
+        files = Dir[File.join(failed_dir, '*.json')]
+
+        if files.empty?
+          @logger.info "No failed submissions to retry"
+          return
+        end
+
+        @logger.info "Retrying #{files.size} failed submissions"
+
+        files.each do |file_path|
+          file_name = File.basename(file_path)
+          pending_path = File.join(@queue_base_path, PENDING_DIR, file_name)
+          FileUtils.mv(file_path, pending_path)
+          @logger.debug "Moved failed submission back to pending: #{file_name}"
+        end
+
+      rescue => e
+        @logger.error "Failed to retry failed submissions: #{e.message}"
+      end
+    end
+
+    # Clean up old success files
+    # @param days_to_keep [Integer] Number of days to keep success files
+    def cleanup_old_success_files(days_to_keep)
+      begin
+        success_dir = File.join(@queue_base_path, SUCCESS_DIR)
+        cutoff_time = Time.now - (days_to_keep * 24 * 60 * 60)
+
+        files = Dir[File.join(success_dir, '*.json')].select do |file|
+          File.mtime(file) < cutoff_time
+        end
+
+        files.each do |file|
+          File.delete(file)
+          @logger.debug "Cleaned up old success file: #{File.basename(file)}"
+        end
+
+        @logger.info "Cleaned up #{files.size} old success files" unless files.empty?
+
+      rescue => e
+        @logger.error "Failed to cleanup old success files: #{e.message}"
+      end
+    end
+
+    # Clear all files from the queue (emergency cleanup)
+    def clear_all_queues
+      begin
+        @logger.info "Clearing all queue directories..."
+
+        clear_directory(PENDING_DIR)
+        clear_directory(PROCESSING_DIR)
+        clear_directory(FAILED_DIR)
+        clear_directory(SUCCESS_DIR)
+
+        @logger.info "All queue directories cleared successfully"
+
+      rescue => e
+        @logger.error "Error clearing queue directories: #{e.message}"
+        raise "Failed to clear queues: #{e.message}"
+      end
+    end
+
+    # Clear all files from a specific queue directory
+    # @param dir_name [String] Directory name to clear
+    private def clear_directory(dir_name)
+      dir = File.join(@queue_base_path, dir_name)
+      if File.directory?(dir)
+        files = Dir[File.join(dir, '*.json')]
+        files.each do |file|
+          File.delete(file)
+          @logger.debug "Deleted file: #{File.basename(file)}"
+        end
+        @logger.info "Cleared #{files.size} files from #{dir_name}"
+      end
+    end
+
+    # Clean up duplicate files across queue directories
+    def cleanup_duplicate_files
+      begin
+        @logger.info "Cleaning up duplicate files across queue directories..."
+
+        file_map = {}
+        [PENDING_DIR, PROCESSING_DIR, FAILED_DIR, SUCCESS_DIR].each do |dir_name|
+          dir = File.join(@queue_base_path, dir_name)
+          if File.directory?(dir)
+            Dir[File.join(dir, '*.json')].each do |file|
+              file_name = File.basename(file)
+              if file_map[file_name]
+                # File exists in multiple directories, keep the one with latest modification time
+                existing_time = File.mtime(file_map[file_name])
+                current_time = File.mtime(file)
+
+                if current_time > existing_time
+                  File.delete(file_map[file_name])
+                  file_map[file_name] = file
+                  @logger.debug "Removed duplicate file (older): #{file_map[file_name]}"
+                else
+                  File.delete(file)
+                  @logger.debug "Removed duplicate file (older): #{file}"
+                end
+              else
+                file_map[file_name] = file
+              end
+            end
+          end
+        end
+
+        @logger.info "Duplicate file cleanup completed"
+
+      rescue => e
+        @logger.error "Error during duplicate file cleanup: #{e.message}"
+      end
+    end
+  end
+end