sumologic-query 1.1.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec844dca9513804180d8e25badb43d663e46467d9683366750131d1c6acc42f7
-  data.tar.gz: 0e5dc7f91e61572866e817ea9c6db84112d1cca3d5156826bfc71d829359f3fb
+  metadata.gz: 7b7bbf8f091fa6d3c6e213a34c1ba72952c43aa995814834379c30a051d3f4f5
+  data.tar.gz: 4e7bc3f985d01543a5af7ce89d7746944107d4fd99ecf4367c276a5e550dbdca
 SHA512:
-  metadata.gz: 1dbd2cf57b915d5a4e4bd0a43b2f911089087e906a4b01cab318dd3901c3d1645271718c1ff112a7d14769074e7a6dc5f4751001538f72ac703c23b4cb58c505
-  data.tar.gz: 20481e3c0b0f2c86b5b06f4f1fc5d32f4753e96defc6cd4f9a35e671666b28e24c1c388390cddb56443c6020c33123e40cdb93424ae25179bb0bafcd825ac60c
+  metadata.gz: 4f34b22f610d453649fff4926c7dfdff0fd2d5ff18a2e5fa6994cf1f41635990c28ace805fb2508520a1f016899c99f71ba31be2d5b0771a8b30d9e781f310a2
+  data.tar.gz: 0a12a47f8ddfeda0bab912f2011a033b121e01f3c63b19d9fa80e7cfa0a7df32a7277784a92a88ca988819a1f42c717c61dbbae79d5bab154161d9cff5f571a8

data/lib/sumologic/metadata/parallel_fetcher.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Metadata
+    # Handles parallel fetching of sources from multiple collectors
+    class ParallelFetcher
+      def initialize(max_threads: 10)
+        @max_threads = max_threads
+      end
+
+      # Fetch sources for collectors in parallel
+      # Returns array of results with collector info and sources
+      def fetch_all(collectors, &block)
+        result = []
+        mutex = Mutex.new
+        queue = create_work_queue(collectors)
+        threads = create_workers(queue, result, mutex, &block)
+
+        threads.each(&:join)
+        result
+      end
+
+      private
+
+      def create_work_queue(collectors)
+        queue = Queue.new
+        collectors.each { |collector| queue << collector }
+        queue
+      end
+
+      def create_workers(queue, result, mutex, &block)
+        worker_count = [@max_threads, queue.size].min
+
+        Array.new(worker_count) do
+          Thread.new { process_queue(queue, result, mutex, &block) }
+        end
+      end
+
+      def process_queue(queue, result, mutex, &block)
+        until queue.empty?
+          collector = pop_safely(queue)
+          break unless collector
+
+          process_collector(collector, result, mutex, &block)
+        end
+      end
+
+      def pop_safely(queue)
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def process_collector(collector, result, mutex, &block)
+        collector_result = block.call(collector)
+
+        mutex.synchronize do
+          result << collector_result if collector_result
+        end
+      end
+    end
+  end
+end

data/lib/sumologic/metadata/source.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'parallel_fetcher'
+
 module Sumologic
   module Metadata
     # Handles source metadata operations
@@ -7,6 +9,7 @@ module Sumologic
       def initialize(http_client:, collector_client:)
         @http = http_client
         @collector_client = collector_client
+        @parallel_fetcher = ParallelFetcher.new(max_threads: 10)
       end
 
       # List sources for a specific collector
@@ -26,30 +29,15 @@ module Sumologic
 
       # List all sources from all collectors
       # Returns array of hashes with collector info and their sources
+      # Uses parallel fetching with thread pool for better performance
       def list_all
         collectors = @collector_client.list
-        result = []
-
-        collectors.each do |collector|
-          next unless collector['alive'] # Skip offline collectors
-
-          collector_id = collector['id']
-          collector_name = collector['name']
+        active_collectors = collectors.select { |c| c['alive'] }
 
-          log_info "Fetching sources for collector: #{collector_name} (#{collector_id})"
+        log_info "Fetching sources for #{active_collectors.size} active collectors in parallel..."
 
-          sources = list(collector_id: collector_id)
-
-          result << {
-            'collector' => {
-              'id' => collector_id,
-              'name' => collector_name,
-              'collectorType' => collector['collectorType']
-            },
-            'sources' => sources
-          }
-        rescue StandardError => e
-          log_error "Failed to fetch sources for collector #{collector_name}: #{e.message}"
+        result = @parallel_fetcher.fetch_all(active_collectors) do |collector|
+          fetch_collector_sources(collector)
         end
 
         log_info "Total: #{result.size} collectors with sources"
@@ -60,6 +48,27 @@ module Sumologic
 
       private
 
+      # Fetch sources for a single collector
+      def fetch_collector_sources(collector)
+        collector_id = collector['id']
+        collector_name = collector['name']
+
+        log_info "Fetching sources for collector: #{collector_name} (#{collector_id})"
+        sources = list(collector_id: collector_id)
+
+        {
+          'collector' => {
+            'id' => collector_id,
+            'name' => collector_name,
+            'collectorType' => collector['collectorType']
+          },
+          'sources' => sources
+        }
+      rescue StandardError => e
+        log_error "Failed to fetch sources for collector #{collector_name}: #{e.message}"
+        nil
+      end
+
       def log_info(message)
         warn "[Sumologic::Metadata::Source] #{message}" if ENV['SUMO_DEBUG'] || $DEBUG
       end

data/lib/sumologic/search/poller.rb

@@ -11,6 +11,7 @@ module Sumologic
 
       # Poll until job completes or times out
       # Returns final job status data
+      # Starts polling immediately, then applies exponential backoff
       def poll(job_id)
         start_time = Time.now
         interval = @config.initial_poll_interval
@@ -32,6 +33,7 @@ module Sumologic
             raise Error, "Search job #{state.downcase}"
           end
 
+          # Sleep after checking status (not before first check)
           sleep interval
           poll_count += 1
           interval = calculate_next_interval(interval)

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.1.2'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.0
 platform: ruby
 authors:
 - patrick204nqh
@@ -102,6 +102,7 @@ files:
 - lib/sumologic/http/authenticator.rb
 - lib/sumologic/http/client.rb
 - lib/sumologic/metadata/collector.rb
+- lib/sumologic/metadata/parallel_fetcher.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
 - lib/sumologic/search/paginator.rb