fluent-plugin-azure-logs-ingestion 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e5482480b26a287f04f3869ea12ddbe976dac865aa507f43538a99fcfc060c51
-  data.tar.gz: 171a565a9d11e5a020e254682540a85adafa8b90eb46e2732819b1a63f67dab6
+  metadata.gz: 2d502da5879c0445acbdf411a24c2af70b7ad9016eb922474eda10f342d919c4
+  data.tar.gz: 2fee3972f2418f56c1cf5823bec48dc0abebe4e8738e108a7f727dca5e4d5b01
 SHA512:
-  metadata.gz: 26314140847581088a5b2fa71e0fb4abeff7de0fae64ce31afcab42e40c5298d93df43def1809b1f0faf442075e57ebb4e796bd78a95989409550ead49da0037
-  data.tar.gz: 5006e5ddbeff5b09a2a2f6d7e79a68e78d1a7912ccf8e390e6aa997c71f97f151a6e0fefeec6b295daae0bbf014f3c361d6bc4c901c3886be0da33a049700da5
+  metadata.gz: 1e64e5b98b5dde247d1a73b9bb437114d87ca39caf79f5bc1dff20264e7980b12d09cfd0e9b8ee748516f9e4e1cccb8c35a5103f281e7fad93454754e348861a
+  data.tar.gz: a48c818655f21c3292fbd5e36327df8870e001935f878fcdd76cae804fb5a2341bac47cbe6647f3f2e6830c41e0cb80630b77d0ba78f0e8bfeb51a3a340a21dd

data/Gemfile

@@ -4,5 +4,8 @@ source 'https://rubygems.org'
 
 gemspec
 
+gem 'fluentd', ENV['FLUENTD_VERSION'] if ENV['FLUENTD_VERSION']
+gem 'csv', '< 3.2' if ENV['FLUENTD_VERSION'] && Gem::Version.new(ENV['FLUENTD_VERSION']) < Gem::Version.new('1.18')
+
 gem 'rake'
 gem 'test-unit'

data/README.md

@@ -5,6 +5,11 @@ Fluentd output plugin that sends records to Log Analytics Workspace tables by us
 > [!WARNING]
 > This plugin is experimental and has not yet been sufficiently proven in serious production workloads.
 
+## Supported Environment
+
+- Ruby 2.4 or later
+- Fluentd 1.13.0 or later, before 2.0
+
 ## Installation
 
 ### RubyGems
@@ -30,7 +35,7 @@ Then run `bundle install`.
 Bundler can point directly at the GitHub repository. Specify `ref` when you want to pin a specific revision.
 
 ```ruby
-gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'abda3b5370ccd61282c8b234ca05042049e09d15'
+gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'dbb0160a8048957dafb3002799dc3e8567da06e6'
 ```
 
 Then run `bundle install`.
@@ -207,3 +212,37 @@ The `time` in `<buffer time>` is the Fluentd event time, not a `time` field insi
 bundle install
 bundle exec rake test
 ```
+
+This project uses a single `Gemfile`. Use rbenv and `FLUENTD_VERSION` when checking older Ruby / Fluentd combinations locally.
+
+```bash
+rbenv init
+source ~/.bashrc
+
+(
+	export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.13.0
+	unset GEM_HOME GEM_PATH MY_RUBY_HOME
+	rbenv install "$RBENV_VERSION" -s
+	rbenv exec gem install bundler -v 2.3.27 --no-document
+	rbenv exec bundle _2.3.27_ install
+	rbenv exec bundle _2.3.27_ exec rake test
+)
+
+(
+	export RBENV_VERSION=2.7.8 FLUENTD_VERSION=1.18.0
+	unset GEM_HOME GEM_PATH MY_RUBY_HOME
+	rbenv install "$RBENV_VERSION" -s
+	rbenv exec gem install bundler -v 2.3.27 --no-document
+	rbenv exec bundle _2.3.27_ install
+	rbenv exec bundle _2.3.27_ exec rake test
+)
+```
+
+Publish one RubyGems artifact. Compatibility is checked with tests; build the distributable gem once with the current Ruby.
+
+```bash
+bundle install
+bundle exec rake test
+bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
+gem push fluent-plugin-azure-logs-ingestion-*.gem
+```

data/README_ja.md

@@ -5,6 +5,11 @@ Azure Monitor Logs Ingestion API を使い、Log Analytics Workspace のテー
 > [!WARNING]
 > この plugin は試験的な実装であり、本格的な production workload での動作実績はまだ十分ではありません。
 
+## サポート環境
+
+- Ruby 2.4 以上
+- Fluentd 1.13.0 以上、2.0 未満
+
 ## インストール
 
 ### RubyGems
@@ -30,7 +35,7 @@ gem 'fluent-plugin-azure-logs-ingestion'
 Bundler で GitHub repository を直接指定でき、特定の revision に固定したい場合は、`ref` を指定します。
 
 ```ruby
-gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'abda3b5370ccd61282c8b234ca05042049e09d15'
+gem 'fluent-plugin-azure-logs-ingestion', git: 'https://github.com/fukasawah/fluent-plugin-azure-logs-ingestion.git', ref: 'dbb0160a8048957dafb3002799dc3e8567da06e6'
 ```
 
 その後、`bundle install` を実行してください。
@@ -206,4 +211,38 @@ Log Analytics Workspace の Auxiliary tier へ送信し、DCR transformation で
 ```bash
 bundle install
 bundle exec rake test
-```
+```
+
+この project は 1 つの `Gemfile` を使います。古い Ruby / Fluentd version の互換性を手元で確認する場合は、rbenv と `FLUENTD_VERSION` を使います。
+
+```bash
+rbenv init
+source ~/.bashrc
+
+(
+	export RBENV_VERSION=2.4.10 FLUENTD_VERSION=1.13.0
+	unset GEM_HOME GEM_PATH MY_RUBY_HOME
+	rbenv install "$RBENV_VERSION" -s
+	rbenv exec gem install bundler -v 2.3.27 --no-document
+	rbenv exec bundle _2.3.27_ install
+	rbenv exec bundle _2.3.27_ exec rake test
+)
+
+(
+	export RBENV_VERSION=2.7.8 FLUENTD_VERSION=1.18.0
+	unset GEM_HOME GEM_PATH MY_RUBY_HOME
+	rbenv install "$RBENV_VERSION" -s
+	rbenv exec gem install bundler -v 2.3.27 --no-document
+	rbenv exec bundle _2.3.27_ install
+	rbenv exec bundle _2.3.27_ exec rake test
+)
+```
+
+RubyGems へ公開する gem は 1 つだけです。互換性は test で確認し、配布する gem は現在の Ruby で build します。
+
+```bash
+bundle install
+bundle exec rake test
+bundle exec gem build fluent-plugin-azure-logs-ingestion.gemspec --strict
+gem push fluent-plugin-azure-logs-ingestion-*.gem
+```

data/lib/fluent/plugin/azure_logs_ingestion/auth.rb

@@ -10,7 +10,7 @@ module Fluent
   module Plugin
     module AzureLogsIngestion
       class Auth
-        Token = Struct.new(:value, :expires_at, keyword_init: true)
+        Token = Struct.new(:value, :expires_at)
 
         IMDS_API_VERSION = '2018-02-01'
         APP_SERVICE_API_VERSION = '2019-08-01'
@@ -159,7 +159,7 @@ module Fluent
         def build_token_from_token_response(body)
           token = body.fetch('access_token')
           expires_at = parse_token_expiry(body)
-          Token.new(value: token, expires_at: expires_at)
+          Token.new(token, expires_at)
         end
 
         def parse_token_expiry(body)

data/lib/fluent/plugin/azure_logs_ingestion/payload_builder.rb

@@ -15,8 +15,7 @@ module Fluent
           :content_length,
           :raw_size,
           :gzip_size,
-          :record_count,
-          keyword_init: true
+          :record_count
         ) do
           def close!
             io.close!
@@ -65,14 +64,7 @@ module Fluent
 
           validate!(raw_size: raw_size, gzip_size: gzip_size)
 
-          Result.new(
-            io: io,
-            content_encoding: content_encoding,
-            content_length: content_length,
-            raw_size: raw_size,
-            gzip_size: gzip_size,
-            record_count: record_count
-          )
+          Result.new(io, content_encoding, content_length, raw_size, gzip_size, record_count)
         rescue StandardError
           gzip_file.close! if gzip_file
           raw_file.close! if raw_file

data/lib/fluent/plugin/azure_logs_ingestion/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module AzureLogsIngestion
-      VERSION = '0.1.0'
+      VERSION = '0.1.2'
     end
   end
 end

data/test/helper.rb

@@ -1,5 +1,14 @@
 # frozen_string_literal: true
 
+if ENV['COVERAGE']
+	require 'simplecov'
+	SimpleCov.enable_coverage :branch if SimpleCov.respond_to?(:enable_coverage)
+	SimpleCov.start do
+		add_filter '/test/'
+		add_filter '/vendor/'
+		track_files 'lib/**/*.rb'
+	end
+end
 require 'test/unit'
 require 'fluent/test'
 

data/test/support/fake_azure_server.rb

@@ -4,7 +4,7 @@ require 'socket'
 require 'thread'
 
 class FakeAzureServer
-  Request = Struct.new(:method, :path, :headers, :body, keyword_init: true)
+  Request = Struct.new(:method, :path, :headers, :body)
 
   attr_reader :requests
 
@@ -61,7 +61,7 @@ class FakeAzureServer
     end
 
     body = read_body(socket, headers)
-    request = Request.new(method: method, path: path, headers: headers, body: body)
+    request = Request.new(method, path, headers, body)
     @mutex.synchronize { @requests << request }
     status, response_headers, response_body = @handler.call(request)
     write_response(socket, status, response_headers || {}, response_body || '')

data/test/support/helpers.rb

@@ -5,10 +5,21 @@ require 'time'
 require 'uri'
 
 class TestLogger
-  def debug(*) = nil
-  def info(*) = nil
-  def warn(*) = nil
-  def error(*) = nil
+  def debug(*)
+    nil
+  end
+
+  def info(*)
+    nil
+  end
+
+  def warn(*)
+    nil
+  end
+
+  def error(*)
+    nil
+  end
 end
 
 FakeChunk = Struct.new(:events, :chunk_id) do

data/test/test_auth_msi.rb

@@ -13,32 +13,101 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
       [200, { 'Content-Type' => 'application/json' }, { access_token: 'msi-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
     end.start
 
-    with_env(
-      'IDENTITY_ENDPOINT' => "#{server.url}/msi/token",
-      'IDENTITY_HEADER' => 'identity-header-value',
-      'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
-    ) do
-      auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
-        use_msi: true,
-        tenant_id: nil,
-        client_id: 'user-assigned-client-id',
-        client_secret: nil,
-        authority_host: 'https://login.microsoftonline.com',
-        logs_ingestion_scope: 'https://monitor.azure.com/.default',
-        token_refresh_skew: 300,
-        logger: TestLogger.new
-      )
-
-      assert_equal 'msi-token', auth.token
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => "#{server.url}/msi/token",
+        'IDENTITY_HEADER' => 'identity-header-value',
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: 'user-assigned-client-id',
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
+
+        assert_equal 'msi-token', auth.token
+      end
+
+      request = server.requests.first
+      query = URI.decode_www_form(URI(request.path).query).to_h
+      assert_equal 'identity-header-value', request.headers['x-identity-header']
+      assert_equal 'https://monitor.azure.com/', query['resource']
+      assert_equal 'user-assigned-client-id', query['client_id']
+    ensure
+      server&.stop
+    end
+  end
+
+  test 'uses app service managed identity secret when header is absent' do
+    server = FakeAzureServer.new do |_request|
+      [200, { 'Content-Type' => 'application/json' }, { access_token: 'msi-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
+    end.start
+
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => "#{server.url}/msi/token?existing=1",
+        'IDENTITY_HEADER' => nil,
+        'MSI_SECRET' => 'msi-secret-value',
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: '',
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
+
+        assert_equal 'msi-token', auth.token
+      end
+
+      request = server.requests.first
+      query = URI.decode_www_form(URI(request.path).query).to_h
+      assert_equal '1', query['existing']
+      assert_equal 'msi-secret-value', request.headers['secret']
+      assert_nil query['client_id']
+    ensure
+      server&.stop
     end
+  end
+
+  test 'rejects invalid app service managed identity json response' do
+    server = FakeAzureServer.new do |_request|
+      [200, { 'Content-Type' => 'application/json' }, '{']
+    end.start
+
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => "#{server.url}/msi/token",
+        'IDENTITY_HEADER' => 'identity-header-value',
+        'MSI_SECRET' => nil,
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => nil
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: nil,
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
 
-    request = server.requests.first
-    query = URI.decode_www_form(URI(request.path).query).to_h
-    assert_equal 'identity-header-value', request.headers['x-identity-header']
-    assert_equal 'https://monitor.azure.com/', query['resource']
-    assert_equal 'user-assigned-client-id', query['client_id']
-  ensure
-    server&.stop
+        error = assert_raise(RuntimeError) { auth.token }
+        assert_match(/failed to parse App Service managed identity response/, error.message)
+      end
+    ensure
+      server&.stop
+    end
   end
 
   test 'uses IMDS endpoint when app service environment is absent' do
@@ -46,31 +115,93 @@ class AuthManagedIdentityTest < Test::Unit::TestCase
       [200, { 'Content-Type' => 'application/json' }, { access_token: 'imds-token', expires_on: (Time.now.to_i + 3600).to_s }.to_json]
     end.start
 
-    with_env(
-      'IDENTITY_ENDPOINT' => nil,
-      'IDENTITY_HEADER' => nil,
-      'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
-    ) do
-      auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
-        use_msi: true,
-        tenant_id: nil,
-        client_id: 'user-assigned-client-id',
-        client_secret: nil,
-        authority_host: 'https://login.microsoftonline.com',
-        logs_ingestion_scope: 'https://monitor.azure.com/.default',
-        token_refresh_skew: 300,
-        logger: TestLogger.new
-      )
-
-      assert_equal 'imds-token', auth.token
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => nil,
+        'IDENTITY_HEADER' => nil,
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: 'user-assigned-client-id',
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
+
+        assert_equal 'imds-token', auth.token
+      end
+
+      request = server.requests.first
+      query = URI.decode_www_form(URI(request.path).query).to_h
+      assert_equal 'true', request.headers['metadata']
+      assert_equal 'https://monitor.azure.com/', query['resource']
+      assert_equal 'user-assigned-client-id', query['client_id']
+    ensure
+      server&.stop
+    end
+  end
+
+  test 'keeps retryable managed identity status retryable' do
+    server = FakeAzureServer.new do |_request|
+      [404, { 'Content-Type' => 'application/json' }, '{"error":"not ready"}']
+    end.start
+
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => nil,
+        'IDENTITY_HEADER' => nil,
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: nil,
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
+
+        error = assert_raise(RuntimeError) { auth.token }
+        assert_match(/404/, error.message)
+      end
+    ensure
+      server&.stop
     end
+  end
+
+  test 'raises unrecoverable error for managed identity client error' do
+    server = FakeAzureServer.new do |_request|
+      [401, { 'Content-Type' => 'application/json' }, '{"error":"unauthorized"}']
+    end.start
 
-    request = server.requests.first
-    query = URI.decode_www_form(URI(request.path).query).to_h
-    assert_equal 'true', request.headers['metadata']
-    assert_equal 'https://monitor.azure.com/', query['resource']
-    assert_equal 'user-assigned-client-id', query['client_id']
-  ensure
-    server&.stop
+    begin
+      with_env(
+        'IDENTITY_ENDPOINT' => nil,
+        'IDENTITY_HEADER' => nil,
+        'AZURE_LOGS_INGESTION_IMDS_ENDPOINT' => "#{server.url}/metadata/identity/oauth2/token"
+      ) do
+        auth = Fluent::Plugin::AzureLogsIngestion::Auth.new(
+          use_msi: true,
+          tenant_id: nil,
+          client_id: nil,
+          client_secret: nil,
+          authority_host: 'https://login.microsoftonline.com',
+          logs_ingestion_scope: 'https://monitor.azure.com/.default',
+          token_refresh_skew: 300,
+          logger: TestLogger.new
+        )
+
+        error = assert_raise(Fluent::UnrecoverableError) { auth.token }
+        assert_match(/401/, error.message)
+      end
+    ensure
+      server&.stop
+    end
   end
 end