fluent-plugin-kusto 0.0.1.beta

data/test/plugin/test_out_kusto_config.rb

@@ -0,0 +1,86 @@
+# rubocop:disable all
+# frozen_string_literal: true
+
+require_relative '../helper'
+require 'fluent/plugin/out_kusto'
+require 'fluent/plugin/conffile'
+
+class KustoOutputConfigTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @base_conf = {
+      'kusto_endpoint' => 'https://example.kusto.windows.net',
+      'database_name' => 'testdb',
+      'table_name' => 'testtable',
+      'buffered' => 'true'
+    }
+  end
+
+  test 'plugin raises error if buffer section present but buffered is false' do
+    conf = "#{config_str(@base_conf.merge('buffered' => 'false'))}\n<buffer>\n  @type memory\n</buffer>\n"
+    assert_raise(Fluent::ConfigError) { create_driver(conf) }
+  end
+
+  test 'plugin raises error when required param is empty string' do
+    %w[kusto_endpoint database_name table_name].each do |param|
+      conf_hash = @base_conf.dup
+      conf_hash[param] = ''
+      conf = config_str(conf_hash)
+      assert_raise(Fluent::ConfigError, "Should fail when #{param} is empty") { create_driver(conf) }
+    end
+  end
+
+  test 'plugin raises error when required param is whitespace only' do
+    %w[kusto_endpoint database_name table_name].each do |param|
+      conf_hash = @base_conf.dup
+      conf_hash[param] = '   '
+      conf = config_str(conf_hash)
+      assert_raise(Fluent::ConfigError, "Should fail when #{param} is whitespace only") { create_driver(conf) }
+    end
+  end
+
+  test 'plugin raises error when azure_cloud is invalid' do
+    assert_raise(ArgumentError) { OutputConfiguration.new(@base_conf.merge('azure_cloud' => 'InvalidCloud').transform_keys(&:to_sym)).validate_configuration }
+  end
+
+  test 'plugin accepts valid managed_identity_client_id' do
+    conf = config_str(
+      @base_conf
+        .merge('auth_type' => 'user_managed_identity')
+        .merge('managed_identity_client_id' => 'valid_id')
+        .merge('endpoint' => @base_conf['kusto_endpoint'])
+        .merge('auth_type' => 'user_managed_identity') # Ensure required field is present
+    )
+    assert_nothing_raised { create_driver(conf) }
+  end
+
+  test 'plugin sets aad_endpoint based on azure_cloud' do
+    clouds = {
+      'AzureCloud' => 'https://login.microsoftonline.com',
+      'AzureChinaCloud' => 'https://login.chinacloudapi.cn',
+      'AzureUSGovernment' => 'https://login.microsoftonline.us'
+    }
+    base_conf_sym = @base_conf.transform_keys(&:to_sym)
+    clouds.each do |cloud, aad_url|
+      config = OutputConfiguration.new(
+        base_conf_sym.merge(
+          azure_cloud: cloud,
+          client_app_id: 'dummy-client-id',
+          client_app_secret: 'dummy-secret',
+          tenant_id: 'dummy-tenant'
+        )
+      )
+      assert_equal(aad_url, config.aad_endpoint)
+    end
+  end
+
+  private
+
+  def config_str(hash)
+    hash.map { |k, v| "#{k} #{v}" }.unshift('@type kusto').join("\n")
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::KustoOutput).configure(conf)
+  end
+end

data/test/plugin/test_out_kusto_format.rb

@@ -0,0 +1,280 @@
+# rubocop:disable all
+# frozen_string_literal: true
+
+require_relative '../helper'
+require 'fluent/plugin/out_kusto'
+
+class KustoOutputFormatTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @driver = Fluent::Test::Driver::Output.new(Fluent::Plugin::KustoOutput).configure(<<-CONF)
+      @type kusto
+      endpoint https://example.kusto.windows.net
+      database_name testdb
+      table_name testtable
+      client_id dummy-client-id
+      client_secret dummy-secret
+      tenant_id dummy-tenant
+      buffered true
+      auth_type aad
+    CONF
+  end
+
+  test 'format includes tag, timestamp, and record fields' do
+    tag = 'mytag'
+    time = Time.utc(2024, 1, 1, 12, 0, 0).to_i
+    record = { 'foo' => 'bar', 'baz' => 1 }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 'mytag', parsed['tag']
+    assert_equal Time.at(time).utc.iso8601, parsed['timestamp']
+    assert_equal({ 'foo' => 'bar', 'baz' => 1 }, parsed['record'])
+  end
+
+  test "format prefers record['tag'] over argument tag" do
+    tag = 'outertag'
+    time = Time.now.to_i
+    record = { 'tag' => 'innertag', 'foo' => 'bar' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 'innertag', parsed['tag']
+  end
+
+  test "format uses record['tag'] over all fallbacks" do
+    tag = 'outer'
+    time = 1
+    record = { 'tag' => 'inner', 'host' => 'hostval', 'user' => 'userval', 'message' => '1.2.3.4' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 'inner', parsed['tag']
+  end
+
+  test "format uses record['time'] over record['timestamp'] and time param" do
+    tag = 't'
+    time = 123
+    record = { 'time' => 't1', 'timestamp' => 't2' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 't1', parsed['timestamp']
+  end
+
+  test "format uses record['timestamp'] over time param" do
+    tag = 't'
+    time = 123
+    record = { 'timestamp' => 't2' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 't2', parsed['timestamp']
+  end
+
+  test 'format falls back to default_tag if no tag or fallback fields' do
+    tag = nil
+    time = nil
+    record = { 'foo' => 'bar' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 'default_tag', parsed['tag']
+  end
+
+  test 'format removes tag and time from record in output' do
+    tag = 't'
+    time = 123
+    record = { 'foo' => 1, 'tag' => 't', 'time' => 'sometime' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_not_include parsed['record'].keys, 'tag'
+    assert_not_include parsed['record'].keys, 'time'
+  end
+
+  test 'format falls back to host, user, or IP in message if tag is missing' do
+    tag = nil
+    time = 0
+    record = { 'host' => 'hostval', 'foo' => 1 }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 'hostval', parsed['tag']
+    record2 = { 'user' => 'userval', 'foo' => 2 }
+    json2 = @driver.instance.format(tag, time, record2)
+    parsed2 = JSON.parse(json2)
+    assert_equal 'userval', parsed2['tag']
+    record3 = { 'message' => '192.168.1.1 something happened', 'foo' => 3 }
+    json3 = @driver.instance.format(tag, time, record3)
+    parsed3 = JSON.parse(json3)
+    assert_equal '192.168.1.1', parsed3['tag']
+  end
+
+  test 'format falls back to date/time in log message if timestamp missing' do
+    tag = 't'
+    time = nil
+    record = { 'foo' => 'bar', 'msg' => '[01/Jan/2024:12:00:00 +0000] log entry' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal '[01/Jan/2024:12:00:00 +0000] log entry', parsed['timestamp']
+  end
+
+  test 'format sets timestamp to empty string if no time info' do
+    tag = 't'
+    time = nil
+    record = { 'foo' => 'bar' }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal '', parsed['timestamp']
+  end
+
+  test 'format uses first date/time pattern found in record' do
+    tag = 't'
+    time = nil
+    record = {
+      'foo' => 'no date here',
+      'bar' => '[01/Jan/2024:12:00:00 +0000] log entry',
+      'baz' => '[02/Feb/2025:13:00:00 +0000] another'
+    }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal '[01/Jan/2024:12:00:00 +0000] log entry', parsed['timestamp']
+  end
+
+  test 'format output is valid JSON and ends with newline' do
+    tag = 't'
+    time = 1
+    record = { 'foo' => 'bar' }
+    json = @driver.instance.format(tag, time, record)
+    assert_nothing_raised { JSON.parse(json) }
+    assert_match(/\n\z/, json)
+  end
+
+  test 'format raises error on circular reference' do
+    tag = 't'
+    time = 1
+    record = {}
+    record['self'] = record
+    assert_raise_with_message(RuntimeError, 'Circular reference detected in record') do
+      @driver.instance.format(tag, time, record)
+    end
+  end
+
+  test "format uses any key containing 'time' or 'date' for timestamp" do
+    tag = 't'
+    time = 123
+    record = { 'event_time' => '2024-01-01T12:00:00Z', 'foo' => 1 }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal '2024-01-01T12:00:00Z', parsed['timestamp']
+
+    record2 = { 'logdate' => '2023-12-31', 'foo' => 2 }
+    json2 = @driver.instance.format(tag, time, record2)
+    parsed2 = JSON.parse(json2)
+    assert_equal '2023-12-31', parsed2['timestamp']
+
+    record3 = { 'sometime' => '', 'foo' => 3 }
+    json3 = @driver.instance.format(tag, time, record3)
+    parsed3 = JSON.parse(json3)
+    # Falls back to time param if key exists but is empty
+    assert_equal Time.at(time).utc.iso8601, parsed3['timestamp']
+  end
+
+  test 'format handles record with JSON string value' do
+    tag = 't'
+    time = 1
+    json_str = '{"foo": "bar", "baz": 1}'
+    record = { 'data' => json_str }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal json_str, parsed['record']['data']
+  end
+
+  test 'format handles record with CSV string value' do
+    tag = 't'
+    time = 1
+    csv_str = "foo,bar,baz\n1,2,3"
+    record = { 'csv_data' => csv_str }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal csv_str, parsed['record']['csv_data']
+  end
+
+  test 'format handles record with nested hash (parsed JSON)' do
+    tag = 't'
+    time = 1
+    nested = { 'foo' => 'bar', 'baz' => 1 }
+    record = { 'nested' => nested }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal nested, parsed['record']['nested']
+  end
+
+  test 'format handles nil record gracefully' do
+    tag = 't'
+    time = 1
+    record = nil
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 't', parsed['tag']
+    assert_equal Time.at(time).utc.iso8601, parsed['timestamp']
+    assert_equal({}, parsed['record'])
+  end
+
+  test 'format handles array record' do
+    tag = 't'
+    time = 1
+    record = [1, 2, 3]
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 't', parsed['tag']
+    assert_equal Time.at(time).utc.iso8601, parsed['timestamp']
+    assert_equal [1, 2, 3], parsed['record']
+  end
+
+  test 'format handles deeply nested hash and array' do
+    tag = 't'
+    time = 1
+    record = { 'a' => { 'b' => [1, { 'c' => 2 }] } }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal({ 'a' => { 'b' => [1, { 'c' => 2 }] } }, parsed['record'])
+  end
+
+  test 'format handles special characters in keys and values' do
+    tag = 't'
+    time = 1
+    record = { "uni\u2603" => "snowman\u2603", "newline" => "line1\nline2" }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal "snowman\u2603", parsed['record']["uni\u2603"]
+    assert_equal "line1\nline2", parsed['record']['newline']
+  end
+
+  test 'format handles very large record' do
+    tag = 't'
+    time = 1
+    record = {}
+    1000.times { |i| record["key#{i}"] = i }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal 1000, parsed['record'].size
+  end
+
+  test 'format handles boolean, nil, and float values' do
+    tag = 't'
+    time = 1
+    record = { 'bool' => true, 'nilval' => nil, 'float' => 1.23 }
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal true, parsed['record']['bool']
+    assert_nil parsed['record']['nilval']
+    assert_in_delta 1.23, parsed['record']['float'], 0.0001
+  end
+
+  test 'format handles empty hash and array' do
+    tag = 't'
+    time = 1
+    record = {}
+    json = @driver.instance.format(tag, time, record)
+    parsed = JSON.parse(json)
+    assert_equal({}, parsed['record'])
+    record2 = []
+    json2 = @driver.instance.format(tag, time, record2)
+    parsed2 = JSON.parse(json2)
+    assert_equal [], parsed2['record']
+  end
+end

data/test/plugin/test_out_kusto_process.rb

@@ -0,0 +1,150 @@
+# rubocop:disable all
+require_relative "../helper"
+require "fluent/test/driver/output"
+require "fluent/plugin/out_kusto.rb"
+require "mocha/test_unit"
+
+class KustoOutputProcessTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @driver = Fluent::Test::Driver::Output.new(Fluent::Plugin::KustoOutput).configure(<<-CONF)
+      @type kusto
+      endpoint https://example.kusto.windows.net
+      database_name testdb
+      table_name testtable
+      client_id dummy-client-id
+      client_secret dummy-secret
+      tenant_id dummy-tenant
+      buffered true
+      auth_type aad
+    CONF
+  end
+
+  def set_mocks(ingester: nil, logger: nil)
+    @driver.instance.instance_variable_set(:@ingester, ingester) if ingester
+    @driver.instance.instance_variable_set(:@logger, logger) if logger
+  end
+
+  def event_stream(arr)
+    Fluent::ArrayEventStream.new(arr)
+  end
+
+  def logger_stub
+    m = mock
+    m.stubs(:debug)
+    m.stubs(:error)
+    m.stubs(:warn)
+    m.stubs(:info)
+    m
+  end
+
+  def ingester_stub
+    m = mock
+    m.stubs(:upload_data_to_blob_and_queue)
+    m
+  end
+
+  test "process logs error but continues on upload failure" do
+    ingester_mock = mock; ingester_mock.stubs(:upload_data_to_blob_and_queue).raises(StandardError, "upload failed")
+    logger_mock = mock; logger_mock.stubs(:debug); logger_mock.expects(:error).at_least_once
+    set_mocks(ingester: ingester_mock, logger: logger_mock)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => "bar"}]])) }
+  end
+
+  test "process handles empty event stream" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([])) }
+  end
+
+  test "process handles non-hash record" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, "not a hash"]])) }
+  end
+
+  test "process handles nil record" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, nil]])) }
+  end
+
+  test "process handles format raising error (circular reference)" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    record = {}; record["self"] = record
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, record]])) }
+  end
+
+  test "process handles upload raising different errors" do
+    ingester_mock = mock; ingester_mock.stubs(:upload_data_to_blob_and_queue).raises(IOError, "io error")
+    logger_mock = logger_stub; logger_mock.expects(:error).at_least_once
+    set_mocks(ingester: ingester_mock, logger: logger_mock)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => "bar"}]])) }
+  end
+
+  test "process continues after multiple failures" do
+    ingester_mock = mock; ingester_mock.stubs(:upload_data_to_blob_and_queue).raises(StandardError, "fail")
+    logger_mock = logger_stub; logger_mock.expects(:error).at_least_once
+    set_mocks(ingester: ingester_mock, logger: logger_mock)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => "bar"}], [Time.now.to_i, {"foo" => "baz"}]])) }
+  end
+
+  test "process handles logger being nil" do
+    set_mocks(ingester: ingester_stub, logger: nil)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => "bar"}]])) }
+  end
+
+  test "process handles empty string time" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([["", {"foo" => "bar"}]])) }
+  end
+
+  test "process handles array as record" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, [1,2,3]]])) }
+  end
+
+  test "process handles very large record" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    big_record = {"foo" => "x" * 100_000}
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, big_record]])) }
+  end
+
+  test "process handles deeply nested record" do
+    set_mocks(ingester: ingester_stub, logger: logger_stub)
+    nested = {"a"=>{"b"=>{"c"=>{"d"=>1}}}}
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, nested]])) }
+  end
+
+  test "process handles binary data in record" do
+    logger_mock = mock; logger_mock.stubs(:debug); logger_mock.stubs(:error); logger_mock.stubs(:warn)
+    set_mocks(ingester: mock, logger: logger_mock)
+    record = {"bin" => "\xFF".b}
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, record]])) }
+  end
+
+  test "process encodes formatted data to UTF-8 and replaces invalid characters" do
+    ingester_mock = mock
+    ingester_mock.stubs(:upload_data_to_blob_and_queue)
+    logger_mock = logger_stub
+    logger_mock.stubs(:error)
+    set_mocks(ingester: ingester_mock, logger: logger_mock)
+    bad_utf8 = "foo\xFFbar".force_encoding("ASCII-8BIT")
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => bad_utf8}]]) ) }
+  end
+
+  test "process handles event stream with mixed valid and invalid events" do
+    logger_mock = logger_stub
+    logger_mock.stubs(:error)
+    set_mocks(ingester: ingester_stub, logger: logger_mock)
+    es = event_stream([[Time.now.to_i, {"foo" => "bar"}], [Time.now.to_i, nil], [Time.now.to_i, "not a hash"]])
+    assert_nothing_raised { @driver.instance.process("test.tag", es) }
+  end
+
+  test "process logs error with correct message on upload failure" do
+    ingester_mock = mock
+    ingester_mock.stubs(:upload_data_to_blob_and_queue).raises(StandardError, "upload failed")
+    logger_mock = mock
+    logger_mock.stubs(:debug)
+    logger_mock.expects(:error).with { |msg| msg.include?("Failed to ingest event to Kusto") && msg.include?("upload failed") }
+    set_mocks(ingester: ingester_mock, logger: logger_mock)
+    assert_nothing_raised { @driver.instance.process("test.tag", event_stream([[Time.now.to_i, {"foo" => "bar"}]])) }
+  end
+end