fluent-plugin-kusto 1.0.0 → 1.1.1.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ba5fad040c3a296f33314e11714338aaa50e9a28874aac512439f80efde8463
-  data.tar.gz: 0e51820d8e83c1d4e5e3dddb2b85f20eb0b662b8aa27343c75166ad3e56cf33a
+  metadata.gz: 3f5182c8d42b84b2e41dfb92f5bd01be66e3d2ebefb6171182c16b464d4af27e
+  data.tar.gz: 81ec9a5a9ae6d3d41daa76c1334104e6c7987655e6c81df158081233394b4340
 SHA512:
-  metadata.gz: 2fc98cdf258348febc73ad4eb67c2ca52dfde4a38a15bfd9896096fd40e465edaaa9b1545b8ac07c8c781b4621edd1a4581133c06ecd1c703847692a55fdc720
-  data.tar.gz: 8e4d1fecd4929047f04e1c2f1034b7ab614c781cfd1a3e8268415e4e31578ccdc0fdd62e9fedafc9b1f5c180d9963099da1faaf251c470aa683472dc7d725b6b
+  metadata.gz: f3e7620f5ec7327efb52897436c4f7a40bb0f0a7d9ac3ee7f0f5f39eccf9260c03e0d6b7211f610e7b85eb9de91ffff05f16008c3636f11ef1f135857b8d72b5
+  data.tar.gz: 2cef3305e66b2062f2d5be1b1fac4692ddf8a550885ab9a04c77bc9f861f56e45f74ce3232a0c6860d88fa1c0817234493a1ce3b813af891291835d07eff20cf

data/README.md

@@ -42,7 +42,7 @@ This plugin allows you to send data from Fluentd to Azure Data Explorer (Kusto)
 ### RubyGems
 
 ```sh
-$ gem install fluent-plugin-kusto --pre
+$ gem install fluent-plugin-kusto 
 ```
 
 ### Bundler
@@ -50,11 +50,9 @@ $ gem install fluent-plugin-kusto --pre
 Add the following line to your Gemfile:
 
 ```ruby
-gem "fluent-plugin-kusto", "~> 0.0.2.beta"
+gem "fluent-plugin-kusto", "~> 1.1.1.beta"
 ```
 
-**Note:** This is a beta release. Use the `--pre` flag with gem install or specify the beta version in your Gemfile.
-
 And then execute:
 
 ```sh
@@ -244,6 +242,60 @@ This approach provides flexibility to transform the generic 3-column format into
 | `azure_cloud` | Azure cloud environment: `AzureCloud`, `AzureChinaCloud`, `AzureUSGovernmentCloud`, `AzureGermanCloud` | `AzureCloud` |
 | `logger_path` | File path for plugin logs. If not set, logs to stdout. | stdout |
 
+## Dynamic Table Name Resolution
+
+The plugin supports dynamic table name resolution using placeholders in the `table_name` parameter. This allows you to route logs to different tables based on the Fluentd tag.
+
+### Supported Placeholders
+
+| Placeholder | Description | Example |
+|------------|-------------|---------|
+| `${tag}` | Full tag name (dots → underscores) | `app.orders.created` → `app_orders_created` |
+| `${tag_parts[N]}` | Nth part of tag (0-indexed) | `${tag_parts[1]}` with `app.orders.created` → `orders` |
+| `${tag_prefix[N]}` | First N parts joined | `${tag_prefix[2]}` with `app.orders.created` → `app_orders` |
+| `${tag_suffix[N]}` | Last N parts joined | `${tag_suffix[2]}` with `app.orders.created` → `orders_created` |
+
+### Usage Examples
+
+**Route by tag part:**
+```conf
+<match custom.**>
+  @type kusto
+  table_name ${tag_parts[1]}
+  # custom.orders.created → orders table
+  # custom.users.signup → users table
+</match>
+```
+
+**Mixed static and dynamic:**
+```conf
+<match app.**>
+  @type kusto
+  table_name logs_${tag_parts[1]}
+  # app.orders → logs_orders table
+</match>
+```
+
+**Multiple placeholders:**
+```conf
+<match **>
+  @type kusto
+  table_name ${tag_parts[0]}_${tag_parts[1]}_events
+  # production.api.requests → production_api_events table
+</match>
+```
+
+**Notes:**
+- All special characters are automatically converted to underscores
+- Consecutive underscores are collapsed to single underscores
+- Static table names (without placeholders) continue to work as before
+- Placeholders are resolved at ingestion time based on the event tag
+
+**Important Limitations:**
+- ⚠️ **Delayed commits not supported with dynamic table names**: The `delayed` commit feature is currently incompatible with placeholder-based table names. When using dynamic table names, ensure `delayed` is set to `false` (the default).
+- ⚠️ **Validate placeholder patterns**: Ensure your placeholder patterns always resolve to non-empty, valid table names for all expected tags. For example, accessing a tag part index that doesn't exist (e.g., `${tag_parts[5]}` for tag `app.orders`) will resolve to `"unknown"` as a fallback.
+- ⚠️ **Empty or nil tags**: If a tag is empty or nil when using placeholders, the plugin will use `"unknown"` as a fallback to prevent ingestion failures.
+
 ### Buffer Configuration (buffered mode only)
 | Key | Description | Default |
 | --- | ----------- | ------- |
@@ -393,7 +445,13 @@ This diagram shows the main components and data flow for the plugin, including c
 
 ## Release Notes
 
-### v0.0.2.beta (Latest)
+### v1.1.1.beta (Latest)
+- **Dynamic table name resolution** - Added support for placeholder-based table name routing using `${tag}`, `${tag_parts[N]}`, `${tag_prefix[N]}`, and `${tag_suffix[N]}`
+- **Enhanced flexibility** - Route logs to different tables based on Fluentd tags without code changes
+- **Backwards compatible** - Static table names continue to work as before
+
+### v1.0.0
+- **Production-ready release** - Stable version with comprehensive testing
 - **Fixed critical authentication initialization bugs** - Resolved `NameError` in ManagedIdentityTokenProvider and WorkloadIdentityTokenProvider
 - **Added comprehensive unit test coverage** - New test suites for authentication providers with 14 test cases and 45+ assertions
 - **Improved E2E test reliability** - Enhanced timeout configurations to handle Azure Kusto ingestion delays (480s-600s timeouts)

data/lib/fluent/plugin/kusto_version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module Kusto
-      VERSION = '1.0.0'
+      VERSION = '1.1.1.beta'
     end
   end
 end

data/lib/fluent/plugin/out_kusto.rb

@@ -63,6 +63,7 @@ module Fluent
         validate_buffer_config(conf)
         validate_delayed_config
         validate_required_params
+        @table_name_template = table_name
       end
 
       def start
@@ -70,7 +71,7 @@ module Fluent
         super
         setup_outconfiguration
         setup_ingester_and_logger
-        @table_name = @outconfiguration&.table_name
+        @table_name_template = @outconfiguration&.table_name
         @database_name = @outconfiguration&.database_name
         @shutdown_called = false
         @deferred_threads = []
@@ -160,12 +161,13 @@ module Fluent
       end
 
       def process(tag, es)
+        resolved_table = resolve_table_name(tag)
         es.each do |time, record|
           formatted = format(tag, time, record).encode('UTF-8', invalid: :replace, undef: :replace, replace: '_')
           safe_tag = tag.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '_').gsub(/[^0-9A-Za-z.-]/,
                                                                                                      '_')
           blob_name = "fluentd_event_#{safe_tag}.json"
-          @ingester.upload_data_to_blob_and_queue(formatted, blob_name, @database_name, @table_name,
+          @ingester.upload_data_to_blob_and_queue(formatted, blob_name, @database_name, resolved_table,
                                                   compression_enabled, @ingestion_mapping_reference)
         rescue StandardError => e
           @logger&.error("Failed to ingest event to Kusto: #{e}\nEvent skipped: #{record.inspect}\n#{e.backtrace.join("\n")}")
@@ -178,6 +180,7 @@ module Fluent
         worker_id = Fluent::Engine.worker_id
         raw_data = chunk.read
         tag = extract_tag_from_metadata(chunk.metadata)
+        resolved_table = resolve_table_name(tag)
         safe_tag = tag.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '_').gsub(/[^0-9A-Za-z.-]/,
                                                                                                    '_')
         unique_id = chunk.unique_id
@@ -185,7 +188,7 @@ module Fluent
         blob_name = "fluentd_event_worker#{worker_id}_#{safe_tag}_#{dump_unique_id_hex(unique_id)}#{ext}"
         data_to_upload = compression_enabled ? compress_data(raw_data) : raw_data
         begin
-          @ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, @table_name,
+          @ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, resolved_table,
                                                   compression_enabled, @ingestion_mapping_reference)
         rescue StandardError => e
           handle_kusto_error(e, unique_id)
@@ -208,6 +211,7 @@ module Fluent
       def try_write(chunk)
         @deferred_threads ||= []
         tag = extract_tag_from_metadata(chunk.metadata)
+        resolved_table = resolve_table_name(tag)
         safe_tag = tag.to_s.encode('UTF-8', invalid: :replace, undef: :replace, replace: '_').gsub(/[^0-9A-Za-z.-]/,
                                                                                                    '_')
         chunk_id = dump_unique_id_hex(chunk.unique_id)
@@ -225,7 +229,7 @@ module Fluent
         row_count = records.size
         data_to_upload = compression_enabled ? compress_data(updated_raw_data) : updated_raw_data
         begin
-          @ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, @table_name,
+          @ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, resolved_table,
                                                   compression_enabled, @ingestion_mapping_reference)
           if @shutdown_called || !@delayed
             commit_write(chunk.unique_id)
@@ -287,9 +291,10 @@ module Fluent
 
       def check_data_on_server(chunk_id, row_count)
         # Query Kusto to verify chunk ingestion
+        # Note: For dynamic table names, this uses the template name which may not work with placeholders
         begin
           # Sanitize inputs to prevent injection attacks
-          safe_table_name = @table_name.to_s.gsub(/[^a-zA-Z0-9_]/, '')
+          safe_table_name = @table_name_template.to_s.gsub(/[^a-zA-Z0-9_${}]/, '')
           safe_chunk_id = chunk_id.to_s.gsub(/[^a-zA-Z0-9_-]/, '')
           query = "#{safe_table_name} | extend record_dynamic = parse_json(record) | where record_dynamic.chunk_id == '#{safe_chunk_id}' | count"
           result = run_kusto_api_query(query, @outconfiguration.kusto_endpoint, @ingester.token_provider,
@@ -340,6 +345,61 @@ module Fluent
         super
       end
 
+      def resolve_table_name(tag)
+        # Resolve table name from template with placeholders
+        if @table_name_template.nil? || @table_name_template.empty?
+          @logger&.error('Table name template is nil or empty')
+          raise Fluent::ConfigError, 'table_name must be set and non-empty'
+        end
+        
+        return @table_name_template unless @table_name_template.include?('${')
+
+        tag_str = tag.to_s
+        
+        # Validate tag is not empty when using placeholders
+        if tag_str.empty?
+          @logger&.warn("Tag is empty when resolving dynamic table name, using template as fallback: #{@table_name_template}")
+          return @table_name_template.gsub(/\$\{[^}]+\}/, 'unknown').gsub(/[^0-9A-Za-z_]/, '_')
+        end
+        
+        tag_parts = tag_str.split('.')
+        result = @table_name_template.dup
+
+        # Replace ${tag} with full tag (dots converted to underscores)
+        result = result.gsub('${tag}', tag_str.gsub('.', '_'))
+
+        # Replace ${tag_parts[N]} with Nth part
+        result = result.gsub(/\$\{tag_parts\[(\d+)\]\}/) do
+          index = ::Regexp.last_match(1).to_i
+          tag_parts[index] || 'unknown'
+        end
+
+        # Replace ${tag_prefix[N]} with first N parts
+        result = result.gsub(/\$\{tag_prefix\[(\d+)\]\}/) do
+          count = ::Regexp.last_match(1).to_i
+          parts = tag_parts.take(count)
+          parts.empty? ? 'unknown' : parts.join('_')
+        end
+
+        # Replace ${tag_suffix[N]} with last N parts
+        result = result.gsub(/\$\{tag_suffix\[(\d+)\]\}/) do
+          count = ::Regexp.last_match(1).to_i
+          parts = tag_parts.last(count)
+          parts.empty? ? 'unknown' : parts.join('_')
+        end
+
+        # Sanitize: replace special characters with underscores and collapse consecutive underscores
+        sanitized = result.gsub(/[^0-9A-Za-z_]/, '_').gsub(/_+/, '_')
+        
+        # Final validation: ensure we don't have an empty table name
+        if sanitized.empty? || sanitized == '_'
+          @logger&.error("Resolved table name is empty or invalid for tag '#{tag}' with template '#{@table_name_template}'")
+          raise Fluent::ConfigError, "table_name resolved to empty or invalid value for tag '#{tag}'"
+        end
+        
+        sanitized
+      end
+
       private
 
       def validate_buffer_config(conf)

data/test/plugin/test_out_kusto_resolve_table_name.rb

@@ -0,0 +1,320 @@
+# rubocop:disable all
+# frozen_string_literal: true
+
+require 'ostruct'
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_kusto'
+require 'mocha/test_unit'
+
+class KustoOutputResolveTableNameTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+  end
+
+  def create_driver(table_name = 'testtable')
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::KustoOutput).configure(<<-CONF)
+      @type kusto
+      endpoint https://example.kusto.windows.net
+      database_name testdb
+      table_name #{table_name}
+      client_id dummy-client-id
+      client_secret dummy-secret
+      tenant_id dummy-tenant
+      buffered true
+      auth_type aad
+    CONF
+  end
+
+  def logger_stub
+    m = mock
+    m.stubs(:debug)
+    m.stubs(:error)
+    m.stubs(:info)
+    m.stubs(:warn)
+    m
+  end
+
+  def ingester_stub
+    m = mock
+    m.stubs(:upload_data_to_blob_and_queue)
+    m
+  end
+
+  def set_mocks(driver, ingester: nil, logger: nil)
+    driver.instance.instance_variable_set(:@ingester, ingester) if ingester
+    driver.instance.instance_variable_set(:@logger, logger) if logger
+  end
+
+  def chunk_stub(data: 'testdata', tag: 'test.tag', unique_id: 'uniqueid'.b, metadata: nil)
+    c = mock
+    c.stubs(:read).returns(data)
+    c.stubs(:metadata).returns(metadata || OpenStruct.new(tag: tag))
+    c.stubs(:unique_id).returns(unique_id)
+    c
+  end
+
+  # ==========================================
+  # resolve_table_name method tests
+  # ==========================================
+
+  test 'resolve_table_name returns static table name when no placeholders' do
+    driver = create_driver('MyStaticTable')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'MyStaticTable', result
+  end
+
+  test 'resolve_table_name replaces ${tag} with full tag' do
+    driver = create_driver('${tag}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'app_orders_created', result  # dots replaced with underscores
+  end
+
+  test 'resolve_table_name replaces ${tag_parts[0]} with first part' do
+    driver = create_driver('${tag_parts[0]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'app', result
+  end
+
+  test 'resolve_table_name replaces ${tag_parts[1]} with second part' do
+    driver = create_driver('${tag_parts[1]}')
+    result = driver.instance.send(:resolve_table_name, 'custom.orders.created')
+    assert_equal 'orders', result
+  end
+
+  test 'resolve_table_name replaces ${tag_parts[2]} with third part' do
+    driver = create_driver('${tag_parts[2]}')
+    result = driver.instance.send(:resolve_table_name, 'custom.orders.created')
+    assert_equal 'created', result
+  end
+
+  test 'resolve_table_name returns empty string for out-of-bounds tag_parts index' do
+    driver = create_driver('${tag_parts[5]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders')
+    assert_equal '', result
+  end
+
+  test 'resolve_table_name replaces ${tag_prefix[1]} with first part' do
+    driver = create_driver('${tag_prefix[1]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'app', result
+  end
+
+  test 'resolve_table_name replaces ${tag_prefix[2]} with first two parts' do
+    driver = create_driver('${tag_prefix[2]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'app_orders', result  # dot replaced with underscore
+  end
+
+  test 'resolve_table_name replaces ${tag_suffix[1]} with last part' do
+    driver = create_driver('${tag_suffix[1]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'created', result
+  end
+
+  test 'resolve_table_name replaces ${tag_suffix[2]} with last two parts' do
+    driver = create_driver('${tag_suffix[2]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.created')
+    assert_equal 'orders_created', result  # dot replaced with underscore
+  end
+
+  test 'resolve_table_name handles mixed static and placeholder' do
+    driver = create_driver('prefix_${tag_parts[1]}_suffix')
+    result = driver.instance.send(:resolve_table_name, 'custom.orders.events')
+    assert_equal 'prefix_orders_suffix', result
+  end
+
+  test 'resolve_table_name handles multiple placeholders' do
+    driver = create_driver('${tag_parts[0]}_${tag_parts[1]}')
+    result = driver.instance.send(:resolve_table_name, 'env.production.logs')
+    assert_equal 'env_production', result
+  end
+
+  test 'resolve_table_name sanitizes special characters' do
+    driver = create_driver('${tag}')
+    result = driver.instance.send(:resolve_table_name, 'app-name.service:type.logs')
+    assert_equal 'app_name_service_type_logs', result  # special chars replaced with underscores
+  end
+
+  test 'resolve_table_name handles single-part tag' do
+    driver = create_driver('${tag_parts[0]}')
+    result = driver.instance.send(:resolve_table_name, 'singletag')
+    assert_equal 'singletag', result
+  end
+
+  test 'resolve_table_name returns empty for tag_parts on single-part tag with index > 0' do
+    driver = create_driver('${tag_parts[1]}')
+    result = driver.instance.send(:resolve_table_name, 'singletag')
+    assert_equal '', result
+  end
+
+  test 'resolve_table_name handles empty tag' do
+    driver = create_driver('${tag}')
+    result = driver.instance.send(:resolve_table_name, '')
+    assert_equal '', result
+  end
+
+  test 'resolve_table_name handles nil tag' do
+    driver = create_driver('${tag}')
+    result = driver.instance.send(:resolve_table_name, nil)
+    assert_equal '', result
+  end
+
+  # ==========================================
+  # Integration tests with write method
+  # ==========================================
+
+  test 'write uses resolved table name with tag_parts placeholder' do
+    driver = create_driver('${tag_parts[1]}')
+    ingester_mock = mock
+    # Verify the table name passed to ingester is 'orders' (from tag custom.orders.events)
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'testdb', db
+      assert_equal 'orders', table
+      true
+    end
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    chunk = chunk_stub(tag: 'custom.orders.events')
+    assert_nothing_raised { driver.instance.write(chunk) }
+  end
+
+  test 'write uses resolved table name with tag placeholder' do
+    driver = create_driver('${tag}')
+    ingester_mock = mock
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'testdb', db
+      assert_equal 'app_service_logs', table  # dots converted to underscores
+      true
+    end
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    chunk = chunk_stub(tag: 'app.service.logs')
+    assert_nothing_raised { driver.instance.write(chunk) }
+  end
+
+  test 'write uses static table name when no placeholders' do
+    driver = create_driver('MyStaticTable')
+    ingester_mock = mock
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'testdb', db
+      assert_equal 'MyStaticTable', table
+      true
+    end
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    chunk = chunk_stub(tag: 'any.tag.here')
+    assert_nothing_raised { driver.instance.write(chunk) }
+  end
+
+  test 'write routes different tags to different tables' do
+    driver = create_driver('${tag_parts[1]}')
+    ingester_mock = mock
+    
+    # First call should use 'orders' table
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'orders', table
+      true
+    end
+    
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    chunk1 = chunk_stub(tag: 'custom.orders.created')
+    assert_nothing_raised { driver.instance.write(chunk1) }
+    
+    # Second call should use 'users' table
+    ingester_mock2 = mock
+    ingester_mock2.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'users', table
+      true
+    end
+    set_mocks(driver, ingester: ingester_mock2, logger: logger_stub)
+    chunk2 = chunk_stub(tag: 'custom.users.signup')
+    assert_nothing_raised { driver.instance.write(chunk2) }
+  end
+
+  # ==========================================
+  # Edge cases and special characters
+  # ==========================================
+
+  test 'resolve_table_name handles tag with numbers' do
+    driver = create_driver('${tag_parts[1]}')
+    result = driver.instance.send(:resolve_table_name, 'app.table123.logs')
+    assert_equal 'table123', result
+  end
+
+  test 'resolve_table_name handles tag with underscores' do
+    driver = create_driver('${tag_parts[1]}')
+    result = driver.instance.send(:resolve_table_name, 'app.my_table.logs')
+    assert_equal 'my_table', result
+  end
+
+  test 'resolve_table_name preserves underscores' do
+    driver = create_driver('my_${tag_parts[1]}_table')
+    result = driver.instance.send(:resolve_table_name, 'app.orders.logs')
+    assert_equal 'my_orders_table', result
+  end
+
+  test 'resolve_table_name handles deeply nested tags' do
+    driver = create_driver('${tag_parts[3]}')
+    result = driver.instance.send(:resolve_table_name, 'level1.level2.level3.level4.level5')
+    assert_equal 'level4', result
+  end
+
+  test 'resolve_table_name with tag_prefix handles more parts than exist' do
+    driver = create_driver('${tag_prefix[10]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders')
+    assert_equal 'app_orders', result  # Returns all parts when count exceeds available
+  end
+
+  test 'resolve_table_name with tag_suffix handles more parts than exist' do
+    driver = create_driver('${tag_suffix[10]}')
+    result = driver.instance.send(:resolve_table_name, 'app.orders')
+    assert_equal 'app_orders', result  # Returns all parts when count exceeds available
+  end
+
+  # ==========================================
+  # Tests for try_write with dynamic table names
+  # ==========================================
+
+  test 'try_write uses resolved table name' do
+    driver = create_driver('${tag_parts[1]}')
+    driver.instance.instance_variable_set(:@delayed, false)
+    driver.instance.instance_variable_set(:@shutdown_called, false)
+    
+    ingester_mock = mock
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'orders', table
+      true
+    end
+    
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    
+    # Need to stub commit_write since we're in non-delayed mode
+    driver.instance.stubs(:commit_write)
+    
+    chunk = chunk_stub(
+      data: '{"tag":"custom.orders","timestamp":"2024-01-01","record":{"key":"value"}}',
+      tag: 'custom.orders.events'
+    )
+    assert_nothing_raised { driver.instance.try_write(chunk) }
+  end
+
+  # ==========================================
+  # Tests for process with dynamic table names
+  # ==========================================
+
+  test 'process uses resolved table name' do
+    driver = create_driver('${tag_parts[1]}')
+    
+    ingester_mock = mock
+    ingester_mock.expects(:upload_data_to_blob_and_queue).once.with do |_data, _blob_name, db, table, _compression, _mapping|
+      assert_equal 'orders', table
+      true
+    end
+    
+    set_mocks(driver, ingester: ingester_mock, logger: logger_stub)
+    
+    # Create a simple event stream
+    es = mock
+    es.stubs(:each).yields(Time.now.to_i, {'message' => 'test'})
+    
+    assert_nothing_raised { driver.instance.process('custom.orders.events', es) }
+  end
+end

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kusto
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1.beta
 platform: ruby
 authors:
 - Komal Rani
 - Kusto OSS IDC Team
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-12-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,21 +149,19 @@ files:
 - test/helper.rb
 - test/plugin/test_azcli_tokenprovider.rb
 - test/plugin/test_e2e_kusto.rb
-- test/plugin/test_mi_tokenprovider.rb
 - test/plugin/test_out_kusto_config.rb
 - test/plugin/test_out_kusto_format.rb
 - test/plugin/test_out_kusto_process.rb
+- test/plugin/test_out_kusto_resolve_table_name.rb
 - test/plugin/test_out_kusto_start.rb
 - test/plugin/test_out_kusto_try_write.rb
 - test/plugin/test_out_kusto_write.rb
-- test/plugin/test_wif_tokenprovider.rb
 homepage: https://github.com/Azure/azure-kusto-fluentd
 licenses:
 - Apache-2.0
 metadata:
   fluentd_plugin: 'true'
   fluentd_group: output
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -179,19 +176,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.7.1
 specification_version: 4
 summary: A custom Fluentd output plugin for Azure Kusto ingestion.
 test_files:
 - test/helper.rb
 - test/plugin/test_azcli_tokenprovider.rb
 - test/plugin/test_e2e_kusto.rb
-- test/plugin/test_mi_tokenprovider.rb
 - test/plugin/test_out_kusto_config.rb
 - test/plugin/test_out_kusto_format.rb
 - test/plugin/test_out_kusto_process.rb
+- test/plugin/test_out_kusto_resolve_table_name.rb
 - test/plugin/test_out_kusto_start.rb
 - test/plugin/test_out_kusto_try_write.rb
 - test/plugin/test_out_kusto_write.rb
-- test/plugin/test_wif_tokenprovider.rb

data/test/plugin/test_mi_tokenprovider.rb

@@ -1,165 +0,0 @@
-# frozen_string_literal: true
-
-require 'test/unit'
-require 'mocha/test_unit'
-require_relative '../../lib/fluent/plugin/auth/mi_tokenprovider'
-
-class DummyConfigForMI
-  attr_reader :kusto_endpoint, :managed_identity_client_id
-
-  def initialize(kusto_endpoint, managed_identity_client_id = nil)
-    @kusto_endpoint = kusto_endpoint
-    @managed_identity_client_id = managed_identity_client_id
-  end
-
-  def logger
-    require 'logger'
-    Logger.new($stdout)
-  end
-end
-
-class ManagedIdentityTokenProviderTest < Test::Unit::TestCase
-  def setup
-    @resource = 'https://kusto.kusto.windows.net'
-    @client_id = '074c3c54-29e2-4230-a81f-333868b8d6ca'
-  end
-
-  def test_initialize_with_user_managed_identity
-    config = DummyConfigForMI.new(@resource, @client_id)
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Verify instance variables are set correctly
-    assert_equal @resource, provider.instance_variable_get(:@resource)
-    assert_equal @client_id, provider.instance_variable_get(:@managed_identity_client_id)
-    assert_equal false, provider.instance_variable_get(:@use_system_assigned)
-    assert_equal true, provider.instance_variable_get(:@use_user_assigned)
-    assert_not_nil provider.instance_variable_get(:@token_acquire_url)
-  end
-
-  def test_initialize_with_system_managed_identity
-    config = DummyConfigForMI.new(@resource, 'SYSTEM')
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Verify instance variables are set correctly for system managed identity
-    assert_equal @resource, provider.instance_variable_get(:@resource)
-    assert_equal 'SYSTEM', provider.instance_variable_get(:@managed_identity_client_id)
-    assert_equal true, provider.instance_variable_get(:@use_system_assigned)
-    assert_equal false, provider.instance_variable_get(:@use_user_assigned)
-    assert_not_nil provider.instance_variable_get(:@token_acquire_url)
-  end
-
-  def test_initialize_with_empty_client_id
-    config = DummyConfigForMI.new(@resource, '')
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Verify instance variables are set correctly for empty client_id
-    assert_equal @resource, provider.instance_variable_get(:@resource)
-    assert_equal '', provider.instance_variable_get(:@managed_identity_client_id)
-    assert_equal false, provider.instance_variable_get(:@use_system_assigned)
-    assert_equal false, provider.instance_variable_get(:@use_user_assigned)
-  end
-
-  def test_token_acquire_url_formation_user_managed_identity
-    config = DummyConfigForMI.new(@resource, @client_id)
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    token_url = provider.instance_variable_get(:@token_acquire_url)
-    expected_base = 'http://169.254.169.254/metadata/identity/oauth2/token'
-    
-    assert token_url.include?(expected_base), "Token URL should contain base IMDS endpoint"
-    assert token_url.include?('resource=https%3A%2F%2Fkusto.kusto.windows.net'), "Token URL should contain encoded resource"
-    assert token_url.include?('api-version=2018-02-01'), "Token URL should contain API version"
-    assert token_url.include?("client_id=#{@client_id}"), "Token URL should contain client_id for user managed identity"
-  end
-
-  def test_token_acquire_url_formation_system_managed_identity
-    config = DummyConfigForMI.new(@resource, 'SYSTEM')
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    token_url = provider.instance_variable_get(:@token_acquire_url)
-    expected_base = 'http://169.254.169.254/metadata/identity/oauth2/token'
-    
-    assert token_url.include?(expected_base), "Token URL should contain base IMDS endpoint"
-    assert token_url.include?('resource=https%3A%2F%2Fkusto.kusto.windows.net'), "Token URL should contain encoded resource"
-    assert token_url.include?('api-version=2018-02-01'), "Token URL should contain API version"
-    assert !token_url.include?('client_id='), "Token URL should NOT contain client_id for system managed identity"
-  end
-
-  def test_fetch_token_success
-    config = DummyConfigForMI.new(@resource, @client_id)
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Mock successful HTTP response
-    mock_response = {
-      'access_token' => 'fake-access-token',
-      'expires_in' => 3600
-    }
-    
-    provider.stubs(:post_token_request).returns(mock_response)
-    
-    result = provider.send(:fetch_token)
-    assert_equal 'fake-access-token', result[:access_token]
-    assert_equal 3600, result[:expires_in]
-  end
-
-  def test_post_token_request_retries_on_failure
-    config = DummyConfigForMI.new(@resource, @client_id)
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Mock HTTP failure followed by success
-    mock_http = mock
-    mock_response_fail = mock
-    mock_response_fail.stubs(:code).returns(500)
-    mock_response_fail.stubs(:body).returns('Internal Server Error')
-    
-    mock_response_success = mock
-    mock_response_success.stubs(:code).returns(200)
-    mock_response_success.stubs(:body).returns('{"access_token":"fake-token","expires_in":3600}')
-    
-    mock_request = mock
-    Net::HTTP::Get.stubs(:new).returns(mock_request)
-    
-    # Mock the HTTP client setup from create_http_client method
-    mock_http.stubs(:use_ssl=)
-    mock_http.stubs(:open_timeout=)
-    mock_http.stubs(:read_timeout=)
-    mock_http.stubs(:write_timeout=)
-    mock_http.expects(:request).twice.returns(mock_response_fail, mock_response_success)
-    Net::HTTP.stubs(:new).returns(mock_http)
-    
-    # Stub sleep to speed up test
-    provider.stubs(:sleep)
-    
-    result = provider.send(:post_token_request)
-    assert_equal 'fake-token', result['access_token']
-  end
-
-  def test_post_token_request_raises_after_max_retries
-    config = DummyConfigForMI.new(@resource, @client_id)
-    provider = ManagedIdentityTokenProvider.new(config)
-    
-    # Mock HTTP failure for all attempts
-    mock_http = mock
-    mock_response_fail = mock
-    mock_response_fail.stubs(:code).returns(500)
-    mock_response_fail.stubs(:body).returns('Internal Server Error')
-    
-    mock_request = mock
-    Net::HTTP::Get.stubs(:new).returns(mock_request)
-    
-    # Mock the HTTP client setup from create_http_client method
-    mock_http.stubs(:use_ssl=)
-    mock_http.stubs(:open_timeout=)
-    mock_http.stubs(:read_timeout=)
-    mock_http.stubs(:write_timeout=)
-    mock_http.stubs(:request).returns(mock_response_fail)
-    Net::HTTP.stubs(:new).returns(mock_http)
-    
-    # Stub sleep to speed up test
-    provider.stubs(:sleep)
-    
-    assert_raise(RuntimeError, 'Failed to get managed identity token after 2 attempts.') do
-      provider.send(:post_token_request)
-    end
-  end
-end

data/test/plugin/test_wif_tokenprovider.rb

@@ -1,145 +0,0 @@
-# frozen_string_literal: true
-
-require 'test/unit'
-require 'mocha/test_unit'
-require_relative '../../lib/fluent/plugin/auth/wif_tokenprovider'
-
-class DummyConfigForWIF
-  attr_reader :kusto_endpoint, :workload_identity_client_id, :workload_identity_tenant_id, :workload_identity_token_file_path
-
-  def initialize(kusto_endpoint, client_id, tenant_id, token_file_path = nil)
-    @kusto_endpoint = kusto_endpoint
-    @workload_identity_client_id = client_id
-    @workload_identity_tenant_id = tenant_id
-    @workload_identity_token_file_path = token_file_path
-  end
-
-  def logger
-    require 'logger'
-    Logger.new($stdout)
-  end
-end
-
-class WorkloadIdentityTest < Test::Unit::TestCase
-  def setup
-    @resource = 'https://kusto.kusto.windows.net'
-    @client_id = '074c3c54-29e2-4230-a81f-333868b8d6ca'
-    @tenant_id = '12345678-1234-1234-1234-123456789abc'
-    @token_file = '/tmp/test-token'
-  end
-
-  def test_initialize_with_custom_token_file
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, @token_file)
-    provider = WorkloadIdentity.new(config)
-    
-    # Verify instance variables are set correctly
-    assert_equal @resource, provider.instance_variable_get(:@kusto_endpoint)
-    assert_equal @client_id, provider.instance_variable_get(:@client_id)
-    assert_equal @tenant_id, provider.instance_variable_get(:@tenant_id)
-    assert_equal @token_file, provider.instance_variable_get(:@token_file)
-    assert_equal "#{@resource}/.default", provider.instance_variable_get(:@scope)
-  end
-
-  def test_initialize_with_default_token_file
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, nil)
-    provider = WorkloadIdentity.new(config)
-    
-    # Verify default token file is used
-    expected_default = '/var/run/secrets/azure/tokens/azure-identity-token'
-    assert_equal expected_default, provider.instance_variable_get(:@token_file)
-  end
-
-  def test_fetch_token_success
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, @token_file)
-    provider = WorkloadIdentity.new(config)
-    
-    # Mock successful token acquisition
-    mock_response = {
-      'access_token' => 'wif-access-token',
-      'expires_in' => 7200
-    }
-    
-    provider.stubs(:acquire_workload_identity_token).returns(mock_response)
-    
-    result = provider.send(:fetch_token)
-    assert_equal 'wif-access-token', result[:access_token]
-    assert_equal 7200, result[:expires_in]
-  end
-
-  def test_acquire_workload_identity_token_success
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, @token_file)
-    provider = WorkloadIdentity.new(config)
-    
-    # Mock file read and HTTP request
-    File.stubs(:read).with(@token_file).returns('fake-oidc-token')
-    
-    mock_response = mock
-    mock_response.stubs(:is_a?).with(Net::HTTPSuccess).returns(true)
-    mock_response.stubs(:body).returns('{"access_token":"wif-token","expires_in":7200}')
-    
-    mock_http = mock
-    mock_http.expects(:use_ssl=).with(true)
-    mock_http.expects(:open_timeout=).with(10)
-    mock_http.expects(:read_timeout=).with(30)
-    mock_http.expects(:write_timeout=).with(10)
-    mock_http.expects(:request).returns(mock_response)
-    
-    Net::HTTP.stubs(:new).returns(mock_http)
-    
-    result = provider.send(:acquire_workload_identity_token)
-    assert_equal 'wif-token', result['access_token']
-    assert_equal 7200, result['expires_in']
-  end
-
-  def test_acquire_workload_identity_token_failure
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, @token_file)
-    provider = WorkloadIdentity.new(config)
-    
-    # Mock file read and HTTP request failure
-    File.stubs(:read).with(@token_file).returns('fake-oidc-token')
-    
-    mock_response = mock
-    mock_response.stubs(:is_a?).with(Net::HTTPSuccess).returns(false)
-    mock_response.stubs(:code).returns(400)
-    mock_response.stubs(:body).returns('Bad Request')
-    
-    mock_http = mock
-    mock_http.expects(:use_ssl=).with(true)
-    mock_http.expects(:open_timeout=).with(10)
-    mock_http.expects(:read_timeout=).with(30)
-    mock_http.expects(:write_timeout=).with(10)
-    mock_http.expects(:request).returns(mock_response)
-    
-    Net::HTTP.stubs(:new).returns(mock_http)
-    
-    assert_raise(RuntimeError, 'Failed to get access token: 400 Bad Request') do
-      provider.send(:acquire_workload_identity_token)
-    end
-  end
-
-  def test_oauth2_endpoint_formation
-    config = DummyConfigForWIF.new(@resource, @client_id, @tenant_id, @token_file)
-    provider = WorkloadIdentity.new(config)
-    
-    # Test that the endpoint is formatted correctly with tenant_id
-    File.stubs(:read).with(@token_file).returns('fake-oidc-token')
-    
-    # Mock the rest of the HTTP request to avoid actual network call
-    mock_response = mock
-    mock_response.stubs(:is_a?).with(Net::HTTPSuccess).returns(true)
-    mock_response.stubs(:body).returns('{"access_token":"test","expires_in":3600}')
-    
-    mock_http = mock
-    mock_http.expects(:use_ssl=).with(true)
-    mock_http.expects(:open_timeout=).with(10)
-    mock_http.expects(:read_timeout=).with(30)
-    mock_http.expects(:write_timeout=).with(10)
-    mock_http.expects(:request).returns(mock_response)
-    
-    Net::HTTP.stubs(:new).returns(mock_http)
-    
-    # Just verify it doesn't crash - the important thing is that setup_config was called
-    result = provider.send(:acquire_workload_identity_token)
-    assert_equal 'test', result['access_token']
-  end
-end