logstash-output-kusto 2.1.6-java → 2.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '06279ebd4aa0183e794b22db79677dd118c06bab02c47f977e8abde6d86d2abf'
-  data.tar.gz: cfcb3597830708d690b1af7752dfc8f45c4b514fc0e9af17addc872ba179ca79
+  metadata.gz: 907d3aff1700d1466d90df2a7a181022f43f70f4e50fa4e51f6517ca2c4824ff
+  data.tar.gz: 6f088a6cca015b0be48dbd1155a52914211ae00402e3d554b94873cb5c8f0ff4
 SHA512:
-  metadata.gz: 392e0e613e5a8b9a19b5ff210cef38d2a914c827d9e0016a550702914ba05fc63bde6d873bb7f5b5f294bc85bd79c735f2188324cd4a63dda0dc4ac8dc512b0d
-  data.tar.gz: 69206742207f87dbef92e8d48cfc111972b6b3c7769ed7de7d71420c5862a010aa3b6ae26373c36dd8ffdf9c2e1edd8e122d46ca28d26a16cc4460ba658aad0a
+  metadata.gz: 3d74753ae4b064d7808a535bb969aa77ce45a4319318af87c26a535bccac992f402de999ce23615501995da1894b651024e672055bb1b225b4267fcbec009266
+  data.tar.gz: d97fee3e3d9e0cf9f82f0575717c24fa258011666ed047b1239bf231da06e5dd0cecd4a5df0b67419185c993fc9c4885ce421624abbb2bc21e9a1bc5a94df4e8

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+# 2.2.0
+
+- Add opt-in streaming ingestion while preserving queued ingestion as the default.
+- Bound streaming requests by encoded bytes without splitting events, and rely on the Kusto SDK for automatic queued fallback.
+- Add atomic durable spooling, restart recovery, stable source identifiers, final-status quarantine, and streaming metrics.
+- Add bounded worker backpressure, interruptible transient retry cycles, graceful shutdown draining, and proxy support.
+- Secure spool ownership and permissions, reject unsafe paths and symlinks, and support Windows directory-fsync behavior.
+- Preserve committed batches and safely handle missing spool files and completed batch-directory cleanup.
 
 # 2.0.3
 

data/README.md

@@ -52,23 +52,62 @@ output {
 ```
 More information about configuring Logstash can be found in the [logstash configuration guide](https://www.elastic.co/guide/en/logstash/current/configuration.html)
 
+### Streaming ingestion
+
+Queued ingestion remains the default and is recommended for high-throughput workloads. For lower ingestion latency, enable streaming mode:
+
+```ruby
+output {
+  kusto {
+    ingestion_mode => "streaming"
+    streaming_max_request_bytes => 1048576
+    streaming_temp_directory => "/var/lib/logstash/kusto-streaming"
+    ingest_url => "https://ingest-<cluster-name>.kusto.windows.net/"
+    app_id => "<application id>"
+    app_key => "<application key/secret>"
+    app_tenant => "<tenant id>"
+    database => "<database name>"
+    table => "<target table>"
+    json_mapping => "<mapping name>"
+  }
+}
+```
+
+Streaming ingestion must be enabled on the target Kusto table. The connector groups encoded events into requests of at most 1 MiB by default, measured in bytes. It never splits one event: an event above the configured target is sent intact, and the Kusto client decides whether to stream it or fall back to queued ingestion.
+
+Before acknowledging a Logstash batch, the connector writes all requests into an atomic local spool batch. Accepted requests are removed, transient failures apply backpressure and retry with interruptible exponential delays, and final non-success statuses are quarantined for investigation without replaying the entire request.
+
+Streaming delivery is **at least once**. A process or host failure after Kusto accepts a request but before the local spool file is removed can cause that request to be recovered and ingested again. The stable source identifier is used for request tracking; it does not provide Kusto-side row deduplication.
+
+The SDK's `Queued` result means the request was transferred to queued ingestion, not that ingestion into the table has completed. Continue monitoring Kusto ingestion failures for later mapping, schema, encoding-policy, or row-size failures.
+
+The default spool is a destination-specific directory under Logstash `path.data`. Streaming mode defaults to private `0700` directories and `0600` files, rejects symlinked or untrusted recovery files, and allows only one active plugin instance per spool. When setting a custom directory, place it on persistent local storage with trusted parent directories. Group- or world-writable `dir_mode` and `file_mode` values are rejected.
+
 ### Available Configuration Keys
 
 | Parameter Name | Description | Notes |
 | --- | --- | --- |
-| **path** | The plugin writes events to temporary files before sending them to ADX. This parameter includes a path where files should be written and a time expression for file rotation to trigger an upload to the ADX service. The example above shows how to rotate the files every minute and check the Logstash docs for more information on time expressions. | Required
+| **path** | The plugin writes events to temporary files before sending them to ADX. This parameter includes a path where files should be written and a time expression for file rotation to trigger an upload to the ADX service. The example above shows how to rotate the files every minute and check the Logstash docs for more information on time expressions. | Required for queued ingestion |
 | **ingest_url** | The Kusto endpoint for ingestion-related communication. See it on the Azure Portal.| Required|
 | **app_id, app_key, app_tenant**| Credentials required to connect to the ADX service. Be sure to use an application with 'ingest' privileges. | Optional|
 | **managed_identity**| Managed Identity to authenticate. For user-based managed ID, use the Client ID GUID. For system-based, use the value `system`. The ID needs to have 'ingest' privileges on the cluster. | Optional|
 | **database**| Database name to place events | Required |
 | **table** | Target table name to place events | Required
 | **json_mapping** | Maps each attribute from incoming event JSON strings to the appropriate column in the table. Note that this must be in JSON format, as this is the interface between Logstash and Kusto | Optional |
-| **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files found in the path upon startup | |
+| **recovery** | If set to true (default), plugin will attempt to resend pre-existing temp files upon startup. With streaming, disabling recovery can leave committed spool files unprocessed after restart. | |
 | **delete_temp_files** | Determines if temp files will be deleted after a successful upload (true is default; set false for debug purposes only)| |
 | **flush_interval** | The time (in seconds) for flushing writes to temporary files. Default is 2 seconds, 0 will flush on every event. Increase this value to reduce IO calls but keep in mind that events in the buffer will be lost in case of abrupt failure.| |
+| **dir_mode** | Directory permissions. Streaming defaults to `0700` and rejects group- or world-writable values. | |
+| **file_mode** | Temporary-file permissions. Streaming defaults to `0600` and rejects group- or world-writable values. | |
 | **proxy_host** | The proxy hostname for redirecting traffic to Kusto.| |
 | **proxy_port** | The proxy port for the proxy. Defaults to 80.| |
 | **proxy_protocol** | The proxy server protocol , is one of http or https.| |
+| **ingestion_mode** | `queued` for throughput-oriented ingestion or `streaming` for low-latency ingestion with automatic queued fallback. | Defaults to `queued` |
+| **streaming_max_request_bytes** | Target maximum encoded bytes per streaming request. A single event is never split. | Defaults to 1048576 |
+| **streaming_max_retry_attempts** | Transient retries before an interruptible cooldown cycle. During an outage, workers remain bounded and backpressure the Logstash pipeline instead of creating an unbounded retry queue. | Defaults to 2 |
+| **streaming_retry_backoff_seconds** | Initial delay for connector retries; later retries use exponential backoff. | Defaults to 1 |
+| **streaming_concurrent_requests** | Maximum streaming upload worker count. | Defaults to 4 |
+| **streaming_temp_directory** | Durable local spool for streaming requests and restart recovery. Only one active output may use a directory. | Defaults below Logstash `path.data/plugins/logstash-output-kusto` |
 
 > Note : LS_JAVA_OPTS can be used to set proxy parameters as well (using export or SET options)
 
@@ -81,6 +120,7 @@ export  LS_JAVA_OPTS="-Dhttp.proxyHost=1.2.34 -Dhttp.proxyPort=8989 -Dhttps.prox
 
 | Version | Release Date | Notes |
 | --- | --- | --- |
+| 2.2.0 | 2026-07-16 | - Add opt-in Kusto streaming ingestion with byte-bounded requests, automatic queued fallback, durable restart recovery, bounded backpressure, secure local spooling, streaming metrics, and production stress coverage. Queued ingestion remains the default. |
 | 2.0.8 | 2024-10-23 | - Fix library deprecations, fix issues in the Azure Identity library  |
 | 2.0.7 | 2024-01-01 | - Update Kusto JAVA SDK  |
 | 2.0.3 | 2023-12-12 | - Make JSON mapping field optional. If not provided logstash output JSON attribute names will be used for column resolution  |

data/lib/logstash/outputs/kusto/ingestor.rb

@@ -11,6 +11,11 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
   class Ingestor
     require 'logstash-output-kusto_jars'
     RETRY_DELAY_SECONDS = 3
+    ACCEPTED_STREAMING_STATUSES = %w[Succeeded Queued Pending].freeze
+    FINAL_STREAMING_STATUSES = %w[Skipped PartiallySucceeded].freeze
+
+    class StreamingIngestionError < StandardError; end
+
     DEFAULT_THREADPOOL = Concurrent::ThreadPoolExecutor.new(
       min_threads: 1,
       max_threads: 8,
@@ -20,9 +25,17 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
     LOW_QUEUE_LENGTH = 3
     FIELD_REF = /%\{[^}]+\}/
 
-    def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, cli_auth, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
+    def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, cli_auth, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL, ingestion_mode = 'queued', streaming_max_retry_attempts = 2, streaming_retry_backoff_seconds = 1, kusto_client = nil, sleeper = nil, streaming_metric = nil)
       @workers_pool = threadpool
       @logger = logger
+      @ingestion_mode = ingestion_mode
+      @streaming_max_retry_attempts = streaming_max_retry_attempts
+      @streaming_retry_backoff_seconds = streaming_retry_backoff_seconds
+      @sleeper = sleeper
+      @streaming_metric = streaming_metric
+      @retry_mutex = Mutex.new
+      @retry_condition = ConditionVariable.new
+      @stopping = false
       validate_config(database, table, json_mapping,proxy_protocol,app_id, app_key, managed_identity_id,cli_auth)
       @logger.info('Preparing Kusto resources.')
 
@@ -64,16 +77,37 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
       # kusto_connection_string.setClientVersionForTracing(name_for_tracing)
       version_for_tracing=Gem.loaded_specs['logstash-output-kusto']&.version || "unknown"
       kusto_connection_string.setConnectorDetails("Logstash",version_for_tracing.to_s,"","",false,"", java_util.Collections.emptyMap());
-      @kusto_client = begin
-        if is_direct_conn
-          kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
+      @kusto_client = kusto_client || begin
+        http_client_properties = unless is_direct_conn
+                                   http_kusto = Java::com.microsoft.azure.kusto.data.http
+                                   java_net = Java::java.net
+                                   proxy_inet_server = java_net.InetSocketAddress.new(proxy_host, proxy_port)
+                                   proxy = Java::com.azure.core.http.ProxyOptions.new(
+                                     Java::com.azure.core.http.ProxyOptions::Type::HTTP,
+                                     proxy_inet_server
+                                   )
+                                   http_kusto.HttpClientProperties.builder().proxy(proxy).build()
+                                 end
+
+        if @ingestion_mode == 'streaming'
+          if http_client_properties
+            kusto_java.ingest.IngestClientFactory.createManagedStreamingIngestClient(
+              kusto_connection_string,
+              http_client_properties,
+              true
+            )
+          else
+            kusto_java.ingest.IngestClientFactory.createManagedStreamingIngestClient(
+              kusto_connection_string
+            )
+          end
+        elsif http_client_properties
+          kusto_java.ingest.IngestClientFactory.createClient(
+            kusto_connection_string,
+            http_client_properties
+          )
         else
-          http_kusto = Java::com.microsoft.azure.kusto.data.http
-          java_net = Java::java.net
-          proxy_inet_server = java_net.InetSocketAddress.new(proxy_host, proxy_port)
-          proxy = Java::com.azure.core.http.ProxyOptions.new(Java::com.azure.core.http.ProxyOptions::Type::HTTP, proxy_inet_server)
-          http_client_properties = http_kusto.HttpClientProperties.builder().proxy(proxy).build()
-          kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string, http_client_properties)
+          kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
         end
       end
 
@@ -132,12 +166,36 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
         LogStash::Util.set_thread_name("Kusto to ingest file: #{path}")
         upload(path, delete_on_success)
       end
-    rescue Exception => e
-      @logger.error('StandardError.', exception: e.class, message: e.message, path: path, backtrace: e.backtrace)
-      raise e
+    rescue => e
+      @logger.error(
+        'Failed to enqueue Kusto ingestion.',
+        exception: e.class,
+        message: e.message,
+        path: path,
+        backtrace: e.backtrace
+      )
+      if @ingestion_mode == 'streaming'
+        return nil if stopping?
+
+        @logger.warn(
+          'Streaming executor rejected a request; ingesting synchronously for backpressure.',
+          path: path
+        )
+        return upload(path, delete_on_success)
+      end
+
+      raise
     end
 
     def upload(path, delete_on_success)
+      return upload_streaming(path, delete_on_success) if
+        @ingestion_mode == 'streaming'
+
+      upload_queued(path, delete_on_success)
+    end
+
+    private
+    def upload_queued(path, delete_on_success)
       file_size = File.size(path)
       @logger.debug("Sending file to kusto: #{path}. size: #{file_size}")
 
@@ -177,9 +235,252 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
       retry
     end
 
+    private
+    def upload_streaming(path, delete_on_success)
+      attempts = 0
+
+      begin
+        file_size = File.size(path)
+        if file_size <= 0
+          @logger.warn("File #{path} is an empty file and is not ingested.")
+          File.delete(path) if delete_on_success
+          return
+        end
+
+        source_id = source_id_for(path)
+        file_source_info =
+          Java::com.microsoft.azure.kusto.ingest.source.FileSourceInfo.new(path, source_id)
+        result = @kusto_client.ingestFromFile(file_source_info, @ingestion_properties)
+        status = validate_streaming_result(result, file_size)
+        if FINAL_STREAMING_STATUSES.include?(status)
+          quarantine_streaming_file(path, status)
+        else
+          complete_streaming_file(path, delete_on_success)
+        end
+        increment_streaming_status(status)
+        status
+      rescue Java::ComMicrosoftAzureKustoIngestExceptions::IngestionClientException => e
+        retain_failed_streaming_file(path, e, 'Streaming request failed permanently.')
+      rescue Java::ComMicrosoftAzureKustoIngestExceptions::IngestionServiceException => e
+        if permanent_service_error?(e)
+          retain_failed_streaming_file(path, e, 'Streaming request failed permanently.')
+          return
+        end
+
+        if attempts >= @streaming_max_retry_attempts
+          delay = @streaming_retry_backoff_seconds * (2**attempts)
+          @streaming_metric.increment(:retry_cycles) unless @streaming_metric.nil?
+          @logger.warn(
+            'Streaming retry cycle was exhausted; applying backpressure before retrying.',
+            path: path,
+            retry_delay_seconds: delay,
+            exception: e.class,
+            message: e.message
+          )
+          attempts = 0
+          return unless wait_for_retry(delay)
+
+          retry
+        end
+
+        delay = @streaming_retry_backoff_seconds * (2**attempts)
+        attempts += 1
+        @logger.warn(
+          'Streaming request failed transiently, retrying.',
+          attempt: attempts,
+          max_retry_attempts: @streaming_max_retry_attempts,
+          retry_delay_seconds: delay,
+          exception: e.class,
+          message: e.message
+        )
+        return unless wait_for_retry(delay)
+
+        retry
+      rescue StreamingIngestionError => e
+        retain_failed_streaming_file(path, e, 'Streaming returned an unsuccessful status.')
+      rescue Errno::ENOENT, Java::JavaNioFile::NoSuchFileException => e
+        @logger.error(
+          "File doesn't exist! Unrecoverable error.",
+          exception: e.class,
+          message: e.message,
+          path: path,
+          backtrace: e.backtrace
+        )
+      rescue => e
+        retain_failed_streaming_file(path, e, 'Streaming request failed unexpectedly.')
+      end
+    end
+
+    private
+    def validate_streaming_result(result, file_size)
+      statuses = result.getIngestionStatusCollection
+      if statuses.nil? || statuses.empty?
+        raise StreamingIngestionError, 'Streaming ingestion returned no status.'
+      end
+
+      ingestion_statuses = statuses.to_a
+      status_names = ingestion_statuses.map { |ingestion_status| ingestion_status.status.to_s }
+      unique_statuses = status_names.uniq
+      if unique_statuses.length > 1
+        @logger.warn(
+          'Streaming request returned mixed statuses and will be quarantined.',
+          statuses: unique_statuses,
+          bytes: file_size
+        )
+        return 'PartiallySucceeded'
+      end
+
+      ingestion_status = ingestion_statuses.first
+      status = unique_statuses.first
+      if FINAL_STREAMING_STATUSES.include?(status)
+        @logger.warn(
+          'Streaming request reached a final non-success status and will not be retried.',
+          status: status,
+          details: ingestion_status.respond_to?(:details) ? ingestion_status.details : nil,
+          error_code: ingestion_status.respond_to?(:errorCode) ? ingestion_status.errorCode : nil,
+          bytes: file_size
+        )
+        return status
+      end
+
+      unless ACCEPTED_STREAMING_STATUSES.include?(status)
+        details = ingestion_status.respond_to?(:details) ? ingestion_status.details : nil
+        raise StreamingIngestionError,
+              "Streaming ingestion returned #{status}. #{details}".strip
+      end
+
+      @logger.debug(
+        "Streaming request accepted. status=#{status} bytes=#{file_size}",
+        status: status,
+        bytes: file_size,
+        database: @ingestion_properties.getDatabaseName,
+        table: @ingestion_properties.getTableName
+      )
+      status
+    end
+
+    private
+    def increment_streaming_status(status)
+      return if @streaming_metric.nil?
+
+      case status
+      when 'Succeeded'
+        @streaming_metric.increment(:streamed)
+      when 'Queued'
+        @streaming_metric.increment(:queued_fallback)
+      when 'Pending'
+        @streaming_metric.increment(:pending)
+      else
+        @streaming_metric.increment(:final_non_success)
+      end
+    end
+
+    private
+    def retain_failed_streaming_file(path, exception, message)
+      @streaming_metric.increment(:failures) unless @streaming_metric.nil?
+      @logger.error(
+        message,
+        exception: exception.class,
+        message: exception.message,
+        path: path,
+        backtrace: exception.backtrace
+      )
+      nil
+    end
+
+    private
+    def source_id_for(path)
+      source_id = File.basename(path)[
+        /stream-\d+-(?<source_id>[0-9a-fA-F-]{36})\.json\z/,
+        :source_id
+      ]
+      return Java::java.util.UUID.fromString(source_id) unless source_id.nil?
+
+      Java::java.util.UUID.nameUUIDFromBytes(File.basename(path).to_java_bytes)
+    end
+
+    private
+    def complete_streaming_file(path, delete_on_success)
+      if delete_on_success
+        File.delete(path)
+      else
+        File.rename(path, "#{path}.completed")
+      end
+
+      parent = File.dirname(path)
+      fsync_directory(parent)
+      return unless File.basename(parent).match?(/\Abatch-.*\.ready\z/)
+
+      Dir.rmdir(parent)
+      fsync_directory(File.dirname(parent))
+    rescue Errno::ENOTEMPTY, Errno::ENOENT
+      nil
+    end
+
+    private
+    def quarantine_streaming_file(path, status)
+      quarantine_path = "#{path}.#{status.downcase}"
+      File.rename(path, quarantine_path)
+      fsync_directory(File.dirname(path))
+      @logger.error(
+        'Streaming request was quarantined after a final non-success status.',
+        status: status,
+        path: quarantine_path
+      )
+    end
+
+    private
+    def fsync_directory(directory)
+      return if Gem.win_platform?
+
+      File.open(directory, File::RDONLY) { |file| file.fsync }
+    rescue SystemCallError => e
+      unsupported_errors = [Errno::EINVAL::Errno]
+      unsupported_errors << Errno::EISDIR::Errno if defined?(Errno::EISDIR)
+      unsupported_errors << Errno::ENOTSUP::Errno if defined?(Errno::ENOTSUP)
+      raise unless unsupported_errors.include?(e.errno)
+
+      @logger.debug('Directory fsync is not supported on this platform.', path: directory)
+    end
+
+    private
+    def wait_for_retry(delay)
+      if @sleeper
+        @sleeper.call(delay)
+        return !stopping?
+      end
+
+      @retry_mutex.synchronize do
+        @retry_condition.wait(@retry_mutex, delay) unless @stopping
+        !@stopping
+      end
+    end
+
+    private
+    def stopping?
+      @retry_mutex.synchronize { @stopping }
+    end
+
+    private
+    def permanent_service_error?(exception)
+      current = exception
+      while current
+        return current.isPermanent if current.respond_to?(:isPermanent)
+
+        current = current.respond_to?(:cause) ? current.cause : nil
+      end
+      false
+    end
+
+    public
     def stop
+      @retry_mutex.synchronize do
+        @stopping = true
+        @retry_condition.broadcast
+      end
       @workers_pool.shutdown
       @workers_pool.wait_for_termination(nil) # block until its done
+      @kusto_client.close
     end
   end
 end

data/lib/logstash/outputs/kusto/streaming_chunker.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require 'logstash/outputs/base'
+require 'logstash/namespace'
+
+class LogStash::Outputs::Kusto < LogStash::Outputs::Base
+  class StreamingChunker
+    def initialize(max_bytes)
+      raise ArgumentError, 'max_bytes must be greater than zero' unless max_bytes.positive?
+
+      @max_bytes = max_bytes
+    end
+
+    def chunks(encoded_events)
+      chunks = []
+      current_chunk = []
+      current_bytes = 0
+
+      encoded_events.each do |encoded|
+        encoded_bytes = encoded.bytesize
+
+        if !current_chunk.empty? && current_bytes + encoded_bytes > @max_bytes
+          chunks << current_chunk
+          current_chunk = []
+          current_bytes = 0
+        end
+
+        current_chunk << encoded
+        current_bytes += encoded_bytes
+
+        if encoded_bytes > @max_bytes
+          chunks << current_chunk
+          current_chunk = []
+          current_bytes = 0
+        end
+      end
+
+      chunks << current_chunk unless current_chunk.empty?
+      chunks
+    end
+  end
+end