logstash-output-kusto 2.0.0-java → 2.0.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 440a318e63597ef2146f2c92410908c958968f37c9cfc2955a1d4796dde0b542
-  data.tar.gz: ff798ffbcd80a6ffe667b52e5b02b33902eeb7df4b539ea8fcd2bfa9e5134d7a
+  metadata.gz: 50f048659e21af84175d7c219ad47e00413f06ca52e6119cd1f03ab6013bd9ac
+  data.tar.gz: e2a60dcd99612c62dc8ee6f0bb053492cf566edcca383d9fc7916707c369450f
 SHA512:
-  metadata.gz: c12afb2dad22c7fb06534905439984df4f083519962e2d298d3c4c47448287941a71f9a6eb885f5f6334c3d51466e3b10c4d53fd92b80c46ec33cfa133fb6e3e
-  data.tar.gz: 3075d4bac3cabd7f53ff1e28b0826151a69f1e9451d29285f021eaceabb2ba9131e8feabe769cb46ad62298471299261b4359a2512608bf9e9c9e2ab5cc8836b
+  metadata.gz: 92252209eb0b2d5f4ecff0a85a08534092f56017272530fd535c66de777f43888c28b039adf1b6e265b15af01135cd077aaa909f0938df583b1890219dde9ee0
+  data.tar.gz: 17a54f5792a3812eeb582d8fb847cfaa27cc7e97a9369888c3b4e4bd4acaa7c9969d98bd39d77fcb50b7a8745c22d1f87323d75300de51d21571f1e688c5bdb0

data/README.md

@@ -53,7 +53,8 @@ More information about configuring Logstash can be found in the [logstash config
 | --- | --- | --- |
 | **path** | The plugin writes events to temporary files before sending them to ADX. This parameter includes a path where files should be written and a time expression for file rotation to trigger an upload to the ADX service. The example above shows how to rotate the files every minute and check the Logstash docs for more information on time expressions. | Required
 | **ingest_url** | The Kusto endpoint for ingestion-related communication. See it on the Azure Portal.| Required|
-| **app_id, app_key, app_tenant**| Credentials required to connect to the ADX service. Be sure to use an application with 'ingest' privileges. | Required|
+| **app_id, app_key, app_tenant**| Credentials required to connect to the ADX service. Be sure to use an application with 'ingest' privileges. | Optional|
+| **managed_identity**| Managed Identity to authenticate. For user-based managed ID, use the Client ID GUID. For system-based, use the value `system`. The ID needs to have 'ingest' privileges on the cluster. | Optional|
 | **database**| Database name to place events | Required |
 | **table** | Target table name to place events | Required
 | **json_mapping** | Maps each attribute from incoming event JSON strings to the appropriate column in the table. Note that this must be in JSON format, as this is the interface between Logstash and Kusto | Required |

data/lib/logstash/outputs/kusto/ingestor.rb

@@ -20,15 +20,34 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
     LOW_QUEUE_LENGTH = 3
     FIELD_REF = /%\{[^}]+\}/
 
-    def initialize(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
+    def initialize(ingest_url, app_id, app_key, app_tenant, managed_identity_id, database, table, json_mapping, delete_local, proxy_host , proxy_port , proxy_protocol,logger, threadpool = DEFAULT_THREADPOOL)
       @workers_pool = threadpool
       @logger = logger
-      validate_config(database, table, json_mapping,proxy_protocol)
+      validate_config(database, table, json_mapping,proxy_protocol,app_id, app_key, managed_identity_id)
       @logger.info('Preparing Kusto resources.')
 
       kusto_java = Java::com.microsoft.azure.kusto
       apache_http = Java::org.apache.http
-      kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
+      # kusto_connection_string = kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
+      # If there is managed identity, use it. This means the AppId and AppKey are empty/nil
+      is_managed_identity = (app_id.nil? && app_key.empty?)
+      # If it is system managed identity, propagate the system identity
+      is_system_assigned_managed_identity = is_managed_identity && 0 == "system".casecmp(managed_identity_id)
+      # Is it direct connection
+      is_direct_conn = (proxy_host.nil? || proxy_host.empty?)
+      # Create a connection string
+      kusto_connection_string = if is_managed_identity
+          if is_system_assigned_managed_identity
+            @logger.info('Using system managed identity.')
+            kusto_java.data.auth.ConnectionStringBuilder.createWithAadManagedIdentity(ingest_url)  
+          else
+            @logger.info('Using user managed identity.')
+            kusto_java.data.auth.ConnectionStringBuilder.createWithAadManagedIdentity(ingest_url, managed_identity_id)
+          end
+        else
+          kusto_java.data.auth.ConnectionStringBuilder.createWithAadApplicationCredentials(ingest_url, app_id, app_key.value, app_tenant)
+        end
+
       #
       @logger.debug(Gem.loaded_specs.to_s)
       # Unfortunately there's no way to avoid using the gem/plugin name directly...
@@ -41,7 +60,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
       kusto_connection_string.setConnectorDetails("Logstash",version_for_tracing.to_s,"","",false,"", tuple_utils.Pair.emptyArray());
       
       @kusto_client = begin
-        if proxy_host.nil? || proxy_host.empty?
+        if is_direct_conn
           kusto_java.ingest.IngestClientFactory.createClient(kusto_connection_string)
         else
           kusto_http_client_properties = kusto_java.data.HttpClientProperties.builder().proxy(apache_http.HttpHost.new(proxy_host,proxy_port,proxy_protocol)).build()
@@ -57,7 +76,12 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
       @logger.debug('Kusto resources are ready.')
     end
 
-    def validate_config(database, table, json_mapping,proxy_protocol)
+    def validate_config(database, table, json_mapping, proxy_protocol, app_id, app_key, managed_identity_id)
+      # Add an additional validation and fail this upfront
+      if app_id.nil? && app_key.empty? && managed_identity_id.empty?
+        @logger.error('managed_identity_id is not provided and app_id/app_key is empty.')
+        raise LogStash::ConfigurationError.new('managed_identity_id is not provided and app_id/app_key is empty.')
+      end      
       if database =~ FIELD_REF
         @logger.error('database config value should not be dynamic.', database)
         raise LogStash::ConfigurationError.new('database config value should not be dynamic.')

data/lib/logstash/outputs/kusto.rb

@@ -73,11 +73,13 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
 
   # The following are the credentails used to connect to the Kusto service
   # application id 
-  config :app_id, validate: :string, required: true
+  config :app_id, validate: :string, required: false
   # application key (secret)
-  config :app_key, validate: :password, required: true
+  config :app_key, validate: :password, required: false
   # aad tenant id
   config :app_tenant, validate: :string, default: nil
+  # managed identity id
+  config :managed_identity, validate: :string, default: nil
 
   # The following are the data settings that impact where events are written to
   # Database name
@@ -150,7 +152,7 @@ class LogStash::Outputs::Kusto < LogStash::Outputs::Base
                                                   max_queue: upload_queue_size,
                                                   fallback_policy: :caller_runs)
 
-    @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor)
+    @ingestor = Ingestor.new(ingest_url, app_id, app_key, app_tenant, managed_identity, database, table, final_mapping, delete_temp_files, proxy_host, proxy_port,proxy_protocol, @logger, executor)
 
     # send existing files
     recover_past_files if recovery

data/lib/logstash-output-kusto_jars.rb

@@ -70,4 +70,4 @@ require_jar('org.ow2.asm', 'asm', '9.3')
 require_jar('org.reactivestreams', 'reactive-streams', '1.0.4')
 require_jar('org.slf4j', 'slf4j-api', '1.8.0-beta4')
 require_jar('org.slf4j', 'slf4j-simple', '1.8.0-beta4')
-require_jar('org.logstash.outputs', 'logstash-output-kusto', '2.0.0')
+require_jar('org.logstash.outputs', 'logstash-output-kusto', '2.0.1')

data/spec/outputs/kusto/ingestor_spec.rb

@@ -9,6 +9,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
   let(:app_id) { "myid" }
   let(:app_key) { LogStash::Util::Password.new("mykey") }
   let(:app_tenant) { "mytenant" }
+  let(:managed_identity) { "managed_identity" }  
   let(:database) { "mydatabase" }
   let(:table) { "mytable" }
   let(:proxy_host) { "localhost" }
@@ -24,7 +25,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
       # note that this will cause an internal error since connection is being tried.
       # however we still want to test that all the java stuff is working as expected
       expect { 
-        ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
+        ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol, logger)
         ingestor.stop
       }.not_to raise_error
     end
@@ -35,7 +36,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
       dynamic_name_array.each do |test_database|
         it "with database: #{test_database}" do
           expect {
-            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity, test_database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
             ingestor.stop
           }.to raise_error(LogStash::ConfigurationError)          
         end
@@ -46,7 +47,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
       dynamic_name_array.each do |test_table|
         it "with database: #{test_table}" do
           expect {
-            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity,database, test_table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
             ingestor.stop
           }.to raise_error(LogStash::ConfigurationError)          
         end
@@ -57,7 +58,7 @@ describe LogStash::Outputs::Kusto::Ingestor do
       dynamic_name_array.each do |json_mapping|
         it "with database: #{json_mapping}" do
           expect {
-            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
+            ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity,database, table, json_mapping, delete_local, proxy_host, proxy_port,proxy_protocol,logger)
             ingestor.stop
           }.to raise_error(LogStash::ConfigurationError)          
         end
@@ -67,7 +68,16 @@ describe LogStash::Outputs::Kusto::Ingestor do
     context 'proxy protocol has to be http or https' do
       it "with proxy protocol: socks" do
         expect {
-          ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
+          ingestor = described_class.new(ingest_url, app_id, app_key, app_tenant, managed_identity,database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
+          ingestor.stop
+        }.to raise_error(LogStash::ConfigurationError)          
+      end
+    end
+
+    context 'one of appid or managedid has to be provided' do
+      it "with empty managed identity and appid" do
+        expect {
+          ingestor = described_class.new(ingest_url, "", app_key, app_tenant, "",database, table, json_mapping, delete_local, proxy_host, proxy_port,'socks',logger)
           ingestor.stop
         }.to raise_error(LogStash::ConfigurationError)          
       end

data/version

@@ -1 +1 @@
-2.0.0
+2.0.1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kusto
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: java
 authors:
 - Tamir Kamara
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-26 00:00:00.000000000 Z
+date: 2023-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -229,7 +229,7 @@ files:
 - vendor/jar-dependencies/org/apache/httpcomponents/httpcore/4.4.16/httpcore-4.4.16.jar
 - vendor/jar-dependencies/org/codehaus/woodstox/stax2-api/4.2.1/stax2-api-4.2.1.jar
 - vendor/jar-dependencies/org/jetbrains/annotations/22.0.0/annotations-22.0.0.jar
-- vendor/jar-dependencies/org/logstash/outputs/logstash-output-kusto/2.0.0/logstash-output-kusto-2.0.0.jar
+- vendor/jar-dependencies/org/logstash/outputs/logstash-output-kusto/2.0.1/logstash-output-kusto-2.0.1.jar
 - vendor/jar-dependencies/org/ow2/asm/asm/9.3/asm-9.3.jar
 - vendor/jar-dependencies/org/reactivestreams/reactive-streams/1.0.4/reactive-streams-1.0.4.jar
 - vendor/jar-dependencies/org/slf4j/slf4j-api/1.8.0-beta4/slf4j-api-1.8.0-beta4.jar