sumologic-query 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b7bbf8f091fa6d3c6e213a34c1ba72952c43aa995814834379c30a051d3f4f5
-  data.tar.gz: 4e7bc3f985d01543a5af7ce89d7746944107d4fd99ecf4367c276a5e550dbdca
+  metadata.gz: 292e0931b6826a0cffd5b28dceac1cdd1b34fcade1112dce48cf067e9e798884
+  data.tar.gz: 9ad954fc938daf22716343d7525b290c1b29b4cdc916915404b0e9315821a62a
 SHA512:
-  metadata.gz: 4f34b22f610d453649fff4926c7dfdff0fd2d5ff18a2e5fa6994cf1f41635990c28ace805fb2508520a1f016899c99f71ba31be2d5b0771a8b30d9e781f310a2
-  data.tar.gz: 0a12a47f8ddfeda0bab912f2011a033b121e01f3c63b19d9fa80e7cfa0a7df32a7277784a92a88ca988819a1f42c717c61dbbae79d5bab154161d9cff5f571a8
+  metadata.gz: 39f7ce4434a5cd6706d3469e98ed264514be13923b13b7265d6912254df60369af5954d2103babdcb394a24dda6ff94e288caff551e651b13411524b9cdff9e8
+  data.tar.gz: 8c718fa5ff299ff50bdfa94636c45e4e1b9fa2f7f3120fe4325b6a4069d18cce61d6f324c0917ab5eacf37fc61b3662c5740055710adc2403b2a238ad13ddffa

data/lib/sumologic/client.rb

@@ -38,6 +38,25 @@ module Sumologic
       )
     end
 
+    # Search logs with streaming interface
+    # Returns an Enumerator that yields messages one at a time
+    # More memory efficient for large result sets
+    #
+    # Example:
+    #   client.search_stream(query: 'error', from_time: ..., to_time: ...).each do |message|
+    #     puts message['map']['message']
+    #   end
+    def search_stream(query:, from_time:, to_time:, time_zone: 'UTC', limit: nil)
+      job_id = @search.create_and_wait(
+        query: query,
+        from_time: from_time,
+        to_time: to_time,
+        time_zone: time_zone
+      )
+
+      @search.stream_messages(job_id, limit: limit)
+    end
+
     # List all collectors
     # Returns array of collector objects
     def list_collectors

data/lib/sumologic/configuration.rb

@@ -4,7 +4,7 @@ module Sumologic
   # Centralized configuration for Sumo Logic client
   class Configuration
     attr_accessor :access_id, :access_key, :deployment, :timeout, :initial_poll_interval, :max_poll_interval,
-                  :poll_backoff_factor, :max_messages_per_request
+                  :poll_backoff_factor, :max_messages_per_request, :enable_parallel_pagination
 
     API_VERSION = 'v1'
 
@@ -22,6 +22,11 @@ module Sumologic
       # Timeouts and limits
       @timeout = 300 # seconds (5 minutes)
       @max_messages_per_request = 10_000
+
+      # Performance options
+      # Parallel pagination enabled by default for better performance
+      # Uses connection pooling for thread-safe concurrent requests
+      @enable_parallel_pagination = true
     end
 
     def base_url

data/lib/sumologic/http/client.rb

@@ -3,27 +3,36 @@
 require 'net/http'
 require 'json'
 require 'uri'
+require_relative 'connection_pool'
 
 module Sumologic
   module Http
     # Handles HTTP communication with Sumo Logic API
     # Responsibilities: request execution, error handling, SSL configuration
+    # Uses connection pooling for thread-safe parallel requests
     class Client
-      READ_TIMEOUT = 60
-      OPEN_TIMEOUT = 10
-
       def initialize(base_url:, authenticator:)
         @base_url = base_url
         @authenticator = authenticator
+        @connection_pool = ConnectionPool.new(base_url: base_url, max_connections: 10)
       end
 
       # Execute HTTP request with error handling
+      # Uses connection pool for thread-safe parallel execution
       def request(method:, path:, body: nil, query_params: nil)
         uri = build_uri(path, query_params)
         request = build_request(method, uri, body)
 
         response = execute_request(uri, request)
         handle_response(response)
+      rescue Errno::ECONNRESET, Errno::EPIPE, EOFError, Net::HTTPBadResponse => e
+        # Connection error - raise for retry at higher level
+        raise Error, "Connection error: #{e.message}"
+      end
+
+      # Close all connections in the pool
+      def close_all_connections
+        @connection_pool.close_all
       end
 
       private
@@ -55,12 +64,9 @@ module Sumologic
       end
 
       def execute_request(uri, request)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true
-        http.read_timeout = READ_TIMEOUT
-        http.open_timeout = OPEN_TIMEOUT
-
-        http.request(request)
+        @connection_pool.with_connection(uri) do |http|
+          http.request(request)
+        end
       end
 
       def handle_response(response)

data/lib/sumologic/http/connection_pool.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Http
+    # Thread-safe connection pool for HTTP clients
+    # Allows multiple threads to have their own connections
+    class ConnectionPool
+      READ_TIMEOUT = 60
+      OPEN_TIMEOUT = 10
+
+      def initialize(base_url:, max_connections: 10)
+        @base_url = base_url
+        @max_connections = max_connections
+        @pool = []
+        @mutex = Mutex.new
+      end
+
+      # Get a connection from the pool (or create new one)
+      def with_connection(uri)
+        connection = acquire_connection(uri)
+        yield connection
+      ensure
+        release_connection(connection) if connection
+      end
+
+      # Close all connections in the pool
+      def close_all
+        @mutex.synchronize do
+          @pool.each do |conn|
+            conn[:http].finish if conn[:http].started?
+          rescue StandardError => e
+            warn "Error closing connection: #{e.message}"
+          end
+          @pool.clear
+        end
+      end
+
+      private
+
+      def acquire_connection(uri)
+        @mutex.synchronize do
+          # Try to find an available connection for this host
+          connection = find_available_connection(uri)
+          return connection[:http] if connection
+
+          # Create new connection if under limit
+          if @pool.size < @max_connections
+            http = create_connection(uri)
+            @pool << { http: http, in_use: true, host: uri.host, port: uri.port }
+            return http
+          end
+
+          # Wait and retry if pool is full
+          nil
+        end || create_temporary_connection(uri)
+      end
+
+      def find_available_connection(uri)
+        connection = @pool.find do |conn|
+          !conn[:in_use] &&
+            conn[:host] == uri.host &&
+            conn[:port] == uri.port &&
+            conn[:http].started?
+        rescue StandardError
+          # Connection is invalid
+          @pool.delete(conn)
+          nil
+        end
+
+        connection[:in_use] = true if connection
+        connection
+      end
+
+      def release_connection(http)
+        @mutex.synchronize do
+          connection = @pool.find { |conn| conn[:http] == http }
+          connection[:in_use] = false if connection
+        end
+      end
+
+      def create_connection(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        http.read_timeout = READ_TIMEOUT
+        http.open_timeout = OPEN_TIMEOUT
+        http.keep_alive_timeout = 30
+        http.start
+        http
+      end
+
+      def create_temporary_connection(uri)
+        # Fallback: create a temporary connection if pool is exhausted
+        create_connection(uri)
+      end
+    end
+  end
+end

data/lib/sumologic/search/job.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative 'stream'
+
 module Sumologic
   module Search
     # Manages search job lifecycle: create, poll, fetch, delete
@@ -9,6 +11,7 @@ module Sumologic
         @config = config
         @poller = Poller.new(http_client: http_client, config: config)
         @paginator = Paginator.new(http_client: http_client, config: config)
+        @stream = Stream.new(paginator: @paginator)
       end
 
       # Execute a complete search workflow
@@ -24,6 +27,22 @@ module Sumologic
         raise Error, "Search failed: #{e.message}"
       end
 
+      # Create job and wait for completion
+      # Returns job_id for use with streaming
+      def create_and_wait(query:, from_time:, to_time:, time_zone: 'UTC')
+        job_id = create(query, from_time, to_time, time_zone)
+        @poller.poll(job_id)
+        job_id
+      end
+
+      # Stream messages from a completed job
+      # Returns an Enumerator
+      def stream_messages(job_id, limit: nil)
+        @stream.each(job_id, limit: limit)
+      ensure
+        delete(job_id)
+      end
+
       private
 
       def create(query, from_time, to_time, time_zone)

data/lib/sumologic/search/paginator.rb

@@ -3,15 +3,40 @@
 module Sumologic
   module Search
     # Handles paginated fetching of search job messages
+    # Supports both sequential and parallel pagination
     class Paginator
+      # Number of pages to fetch in parallel
+      PARALLEL_BATCH_SIZE = 5
+
       def initialize(http_client:, config:)
         @http = http_client
         @config = config
       end
 
       # Fetch all messages for a job with automatic pagination
+      # Uses parallel fetching for better performance on large result sets (if enabled)
       # Returns array of message objects
       def fetch_all(job_id, limit: nil)
+        # Check if parallel pagination is enabled and appropriate
+        if should_use_parallel?(limit)
+          fetch_parallel(job_id, limit: limit)
+        else
+          fetch_sequential(job_id, limit: limit)
+        end
+      end
+
+      private
+
+      # Check if we should use parallel fetching
+      def should_use_parallel?(limit)
+        return false unless @config.enable_parallel_pagination
+
+        # Only use parallel for large result sets (over 20K messages / 2 pages)
+        !limit || limit >= @config.max_messages_per_request * 2
+      end
+
+      # Sequential fetching (original implementation)
+      def fetch_sequential(job_id, limit: nil)
         messages = []
         offset = 0
         total_fetched = 0
@@ -35,7 +60,85 @@ module Sumologic
         messages
       end
 
-      private
+      # Parallel fetching for large result sets
+      def fetch_parallel(job_id, limit: nil)
+        messages = []
+        total_fetched = 0
+
+        loop do
+          pages_to_fetch = calculate_parallel_pages(limit, total_fetched)
+          break if pages_to_fetch.empty?
+
+          batches = fetch_batches_parallel(job_id, pages_to_fetch)
+          total_fetched = process_batches(batches, messages, total_fetched)
+
+          break if done_fetching?(batches, limit, total_fetched)
+        end
+
+        messages
+      end
+
+      # Process fetched batches and update counters
+      def process_batches(batches, messages, total_fetched)
+        batches.each do |batch|
+          messages.concat(batch[:messages])
+          total_fetched += batch[:messages].size
+        end
+
+        log_progress(batches.sum { |b| b[:messages].size }, total_fetched)
+        total_fetched
+      end
+
+      # Check if we're done fetching messages
+      def done_fetching?(batches, limit, total_fetched)
+        last_batch = batches.last
+        return true if last_batch[:messages].size < last_batch[:limit]
+        return true if limit && total_fetched >= limit
+
+        false
+      end
+
+      # Calculate which pages to fetch in parallel
+      def calculate_parallel_pages(limit, total_fetched)
+        pages = []
+        offset = total_fetched
+
+        PARALLEL_BATCH_SIZE.times do
+          batch_limit = calculate_batch_limit(limit, offset)
+          break if batch_limit <= 0
+
+          pages << { offset: offset, limit: batch_limit }
+          offset += batch_limit
+
+          break if limit && offset >= limit
+        end
+
+        pages
+      end
+
+      # Fetch multiple batches in parallel
+      def fetch_batches_parallel(job_id, pages)
+        results = []
+        mutex = Mutex.new
+        threads = pages.map do |page|
+          Thread.new do
+            batch_messages = fetch_batch(job_id, page[:offset], page[:limit])
+
+            mutex.synchronize do
+              results << {
+                offset: page[:offset],
+                limit: page[:limit],
+                messages: batch_messages
+              }
+            end
+          end
+        end
+
+        threads.each(&:join)
+
+        # Sort by offset to maintain order
+        results.sort_by { |r| r[:offset] }
+      end
 
       def calculate_batch_limit(user_limit, total_fetched)
         if user_limit

data/lib/sumologic/search/stream.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Sumologic
+  module Search
+    # Provides streaming interface for search results
+    # Returns an Enumerator that yields messages as they are fetched
+    # Reduces memory usage by not loading all results at once
+    class Stream
+      def initialize(paginator:)
+        @paginator = paginator
+      end
+
+      # Create an enumerator that streams messages from a job
+      # Yields messages one at a time as pages are fetched
+      def each(job_id, limit: nil, &block)
+        return enum_for(:each, job_id, limit: limit) unless block_given?
+
+        stream_messages(job_id, limit: limit, &block)
+      end
+
+      private
+
+      def stream_messages(job_id, limit: nil)
+        offset = 0
+        total_yielded = 0
+
+        loop do
+          batch_limit = calculate_batch_limit(limit, total_yielded)
+          break if batch_limit <= 0
+
+          batch = fetch_batch(job_id, offset, batch_limit)
+          break if batch.empty?
+
+          total_yielded = yield_batch_messages(batch, total_yielded, limit, &Proc.new)
+
+          break if done_streaming?(batch, batch_limit, limit, total_yielded)
+
+          offset += batch.size
+        end
+      end
+
+      # Yield messages from batch and return updated count
+      def yield_batch_messages(batch, total_yielded, limit)
+        batch.each do |message|
+          yield message
+          total_yielded += 1
+          break if limit_reached?(limit, total_yielded)
+        end
+        total_yielded
+      end
+
+      # Check if we've reached the limit
+      def limit_reached?(limit, total_yielded)
+        limit && total_yielded >= limit
+      end
+
+      # Check if we're done streaming
+      def done_streaming?(batch, batch_limit, limit, total_yielded)
+        return true if batch.size < batch_limit # No more messages
+        return true if limit_reached?(limit, total_yielded)
+
+        false
+      end
+
+      def calculate_batch_limit(user_limit, total_yielded)
+        page_size = @paginator.instance_variable_get(:@config).max_messages_per_request
+
+        if user_limit
+          [page_size, user_limit - total_yielded].min
+        else
+          page_size
+        end
+      end
+
+      def fetch_batch(job_id, offset, limit)
+        @paginator.send(:fetch_batch, job_id, offset, limit)
+      end
+    end
+  end
+end

data/lib/sumologic/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sumologic
-  VERSION = '1.2.0'
+  VERSION = '1.2.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sumologic-query
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - patrick204nqh
@@ -101,12 +101,14 @@ files:
 - lib/sumologic/configuration.rb
 - lib/sumologic/http/authenticator.rb
 - lib/sumologic/http/client.rb
+- lib/sumologic/http/connection_pool.rb
 - lib/sumologic/metadata/collector.rb
 - lib/sumologic/metadata/parallel_fetcher.rb
 - lib/sumologic/metadata/source.rb
 - lib/sumologic/search/job.rb
 - lib/sumologic/search/paginator.rb
 - lib/sumologic/search/poller.rb
+- lib/sumologic/search/stream.rb
 - lib/sumologic/version.rb
 homepage: https://github.com/patrick204nqh/sumologic-query
 licenses: