fluent-plugin-azure-logs-ingestion 0.1.0

data/test/test_out_azure_logs_ingestion_write.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require_relative 'support/fake_azure_server'
+require_relative 'support/helpers'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_azure_logs_ingestion'
+require 'stringio'
+require 'zlib'
+
+class AzureLogsIngestionWriteTest < Test::Unit::TestCase
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::AzureLogsIngestionOutput).configure(conf)
+  end
+
+  test 'writes a chunk with service principal auth' do
+    server = FakeAzureServer.new do |request|
+      case request.path
+      when %r{/tenant/oauth2/v2.0/token}
+        [200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
+      when %r{/dataCollectionRules/.+/streams/Custom-MyTable\?api-version=2023-01-01}
+        [200, { 'Content-Type' => 'application/json' }, '{}']
+      else
+        [404, {}, 'not found']
+      end
+    end.start
+
+    driver = create_driver(<<~CONFIG)
+      endpoint #{server.url}
+      authority_host #{server.url}
+      dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
+      stream_name Custom-MyTable
+      tenant_id tenant
+      client_id client
+      client_secret secret
+      <buffer>
+        @type memory
+      </buffer>
+    CONFIG
+
+    driver.instance.start
+    driver.instance.write(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+    ]))
+
+    ingestion_request = server.requests.last
+    assert_equal 'Bearer token-1', ingestion_request.headers['authorization']
+    assert_match(/"message":"hello"/, ingestion_request.body)
+    assert_not_match(/"TimeGenerated":/, ingestion_request.body)
+  ensure
+    driver&.instance&.shutdown
+    driver&.instance&.close
+    server&.stop
+  end
+
+  test 'raises unrecoverable error for 400 response' do
+    server = FakeAzureServer.new do |request|
+      case request.path
+      when %r{/tenant/oauth2/v2.0/token}
+        [200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
+      else
+        [400, { 'Content-Type' => 'application/json' }, '{"error":"bad request"}']
+      end
+    end.start
+
+    driver = create_driver(<<~CONFIG)
+      endpoint #{server.url}
+      authority_host #{server.url}
+      dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
+      stream_name Custom-MyTable
+      tenant_id tenant
+      client_id client
+      client_secret secret
+      <buffer>
+        @type memory
+      </buffer>
+    CONFIG
+
+    driver.instance.start
+    error = assert_raise(Fluent::UnrecoverableError) do
+      driver.instance.write(FakeChunk.new([
+        [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+      ]))
+    end
+
+    assert_match(/400/, error.message)
+  ensure
+    driver&.instance&.shutdown
+    driver&.instance&.close
+    server&.stop
+  end
+
+  test 'sends gzip payload when enabled' do
+    server = FakeAzureServer.new do |request|
+      case request.path
+      when %r{/tenant/oauth2/v2.0/token}
+        [200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
+      when %r{/dataCollectionRules/.+/streams/Custom-MyTable\?api-version=2023-01-01}
+        [200, { 'Content-Type' => 'application/json' }, '{}']
+      else
+        [404, {}, 'not found']
+      end
+    end.start
+
+    driver = create_driver(<<~CONFIG)
+      endpoint #{server.url}
+      authority_host #{server.url}
+      dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
+      stream_name Custom-MyTable
+      tenant_id tenant
+      client_id client
+      client_secret secret
+      gzip true
+      <buffer>
+        @type memory
+      </buffer>
+    CONFIG
+
+    driver.instance.start
+    driver.instance.write(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+    ]))
+
+    ingestion_request = server.requests.last
+    json = Zlib::GzipReader.new(StringIO.new(ingestion_request.body)).read
+    assert_equal 'gzip', ingestion_request.headers['content-encoding']
+    assert_match(/"message":"hello"/, json)
+  ensure
+    driver&.instance&.shutdown
+    driver&.instance&.close
+    server&.stop
+  end
+
+  test 'keeps 429 retryable' do
+    server = FakeAzureServer.new do |request|
+      case request.path
+      when %r{/tenant/oauth2/v2.0/token}
+        [200, { 'Content-Type' => 'application/json' }, { access_token: 'token-1', expires_in: '3600' }.to_json]
+      else
+        [429, { 'Retry-After' => '5' }, '{"error":"throttled"}']
+      end
+    end.start
+
+    driver = create_driver(<<~CONFIG)
+      endpoint #{server.url}
+      authority_host #{server.url}
+      dcr_immutable_id dcr-000a00a000a00000a000000aa000a0aa
+      stream_name Custom-MyTable
+      tenant_id tenant
+      client_id client
+      client_secret secret
+      <buffer>
+        @type memory
+      </buffer>
+    CONFIG
+
+    driver.instance.start
+    error = assert_raise(RuntimeError) do
+      driver.instance.write(FakeChunk.new([
+        [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+      ]))
+    end
+
+    assert_match(/429/, error.message)
+  ensure
+    driver&.instance&.shutdown
+    driver&.instance&.close
+    server&.stop
+  end
+end

data/test/test_payload_builder.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+require_relative 'support/helpers'
+require 'fluent/plugin/azure_logs_ingestion/payload_builder'
+require 'zlib'
+
+class PayloadBuilderTest < Test::Unit::TestCase
+  test 'does not inject TimeGenerated by default' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    result = builder.build(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+    ]))
+
+    body = result.io.read
+    assert_match(/"message":"hello"/, body)
+    assert_not_match(/"TimeGenerated":/, body)
+  ensure
+    result&.close!
+  end
+
+  test 'builds payload from event time' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    result = builder.build(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+    ]))
+
+    body = result.io.read
+    assert_match(/"message":"hello"/, body)
+    assert_equal 1, result.record_count
+    assert_equal result.raw_size, result.content_length
+  ensure
+    result&.close!
+  end
+
+  test 'accepts wide time span' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    result = builder.build(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), {}],
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 31, 0)), {}]
+    ]))
+
+    assert_equal 2, result.record_count
+    result&.close!
+  end
+
+  test 'builds gzip payload when enabled' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: true)
+
+    result = builder.build(FakeChunk.new([
+      [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'hello' }]
+    ]))
+
+    json = Zlib::GzipReader.new(result.io).read
+    assert_equal 'gzip', result.content_encoding
+    assert_match(/"message":"hello"/, json)
+    assert_not_nil result.gzip_size
+  ensure
+    result&.io&.rewind
+    result&.close!
+  end
+
+  test 'rejects oversized payload' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    error = assert_raise(Fluent::UnrecoverableError) do
+      builder.build(FakeChunk.new([
+        [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'a' * 1_100_000 }]
+      ]))
+    end
+
+    assert_match(/payload size/, error.message)
+  end
+
+  test 'accepts non-parseable event time when payload is otherwise valid' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    result = builder.build(FakeChunk.new([
+      ['not-a-time', { 'message' => 'hello' }]
+    ]))
+
+    assert_match(/"message":"hello"/, result.io.read)
+  ensure
+    result&.close!
+  end
+
+  test 'accepts payload records with existing TimeGenerated' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: false)
+
+    result = builder.build(FakeChunk.new([
+      ['not-a-time', { 'message' => 'hello', 'TimeGenerated' => '2026-01-01T00:00:00Z' }]
+    ]))
+
+    body = result.io.read
+    assert_match(/"TimeGenerated":"2026-01-01T00:00:00Z"/, body)
+  ensure
+    result&.close!
+  end
+
+  test 'rejects oversized gzip payload' do
+    builder = Fluent::Plugin::AzureLogsIngestion::PayloadBuilder.new(gzip: true)
+
+    error = assert_raise(Fluent::UnrecoverableError) do
+      builder.build(FakeChunk.new([
+        [Fluent::EventTime.from_time(Time.utc(2026, 1, 1, 0, 0, 0)), { 'message' => 'a' * 1_100_000 }]
+      ]))
+    end
+
+    assert_match(/payload size|gzip payload size/, error.message)
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azure-logs-ingestion
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- fukasawah
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.16'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+description: Buffered Fluentd output plugin for Azure Monitor Logs Ingestion API.
+email:
+- ''
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- README_ja.md
+- Rakefile
+- lib/fluent/plugin/azure_logs_ingestion/auth.rb
+- lib/fluent/plugin/azure_logs_ingestion/client.rb
+- lib/fluent/plugin/azure_logs_ingestion/payload_builder.rb
+- lib/fluent/plugin/azure_logs_ingestion/version.rb
+- lib/fluent/plugin/out_azure_logs_ingestion.rb
+- test/helper.rb
+- test/support/fake_azure_server.rb
+- test/support/helpers.rb
+- test/test_auth_msi.rb
+- test/test_auth_service_principal.rb
+- test/test_out_azure_logs_ingestion.rb
+- test/test_out_azure_logs_ingestion_write.rb
+- test/test_payload_builder.rb
+homepage: https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion
+licenses:
+- Apache-2.0
+metadata:
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Fluentd output plugin for Azure Monitor Logs Ingestion API
+test_files: []