fluent-plugin-kusto 0.0.1.beta → 0.0.3.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/README.md +260 -57
- data/lib/fluent/plugin/auth/aad_tokenprovider.rb +2 -3
- data/lib/fluent/plugin/auth/mi_tokenprovider.rb +8 -7
- data/lib/fluent/plugin/auth/tokenprovider_base.rb +259 -10
- data/lib/fluent/plugin/auth/wif_tokenprovider.rb +22 -3
- data/lib/fluent/plugin/client.rb +82 -1
- data/lib/fluent/plugin/conffile.rb +1 -1
- data/lib/fluent/plugin/ingester.rb +27 -11
- data/lib/fluent/plugin/kusto_constants.rb +57 -0
- data/lib/fluent/plugin/kusto_query.rb +8 -1
- data/lib/fluent/plugin/kusto_version.rb +9 -0
- data/lib/fluent/plugin/out_kusto.rb +5 -3
- data/test/plugin/test_e2e_kusto.rb +459 -119
- data/test/plugin/test_mi_tokenprovider.rb +165 -0
- data/test/plugin/test_wif_tokenprovider.rb +145 -0
- metadata +38 -15
|
@@ -33,6 +33,8 @@ module Fluent
|
|
|
33
33
|
config_param :azure_cloud, :string, default: 'AzureCloud'
|
|
34
34
|
config_param :compression_enabled, :bool, default: true
|
|
35
35
|
config_param :logger_path, :string, default: nil
|
|
36
|
+
config_param :ingestion_mapping_reference, :string, default: nil,
|
|
37
|
+
desc: 'Name of a pre-defined ingestion mapping in Kusto for data transformation during ingestion.'
|
|
36
38
|
config_param :auth_type, :string, default: 'aad',
|
|
37
39
|
desc: 'Authentication type to use for Kusto. Options: "aad", "user_managed_identity", "system_managed_identity", "workload_identity".'
|
|
38
40
|
config_param :workload_identity_client_id, :string, default: nil, secret: true,
|
|
@@ -164,7 +166,7 @@ module Fluent
|
|
|
164
166
|
'_')
|
|
165
167
|
blob_name = "fluentd_event_#{safe_tag}.json"
|
|
166
168
|
@ingester.upload_data_to_blob_and_queue(formatted, blob_name, @database_name, @table_name,
|
|
167
|
-
compression_enabled)
|
|
169
|
+
compression_enabled, @ingestion_mapping_reference)
|
|
168
170
|
rescue StandardError => e
|
|
169
171
|
@logger&.error("Failed to ingest event to Kusto: #{e}\nEvent skipped: #{record.inspect}\n#{e.backtrace.join("\n")}")
|
|
170
172
|
next
|
|
@@ -184,7 +186,7 @@ module Fluent
|
|
|
184
186
|
data_to_upload = compression_enabled ? compress_data(raw_data) : raw_data
|
|
185
187
|
begin
|
|
186
188
|
@ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, @table_name,
|
|
187
|
-
compression_enabled)
|
|
189
|
+
compression_enabled, @ingestion_mapping_reference)
|
|
188
190
|
rescue StandardError => e
|
|
189
191
|
handle_kusto_error(e, unique_id)
|
|
190
192
|
end
|
|
@@ -224,7 +226,7 @@ module Fluent
|
|
|
224
226
|
data_to_upload = compression_enabled ? compress_data(updated_raw_data) : updated_raw_data
|
|
225
227
|
begin
|
|
226
228
|
@ingester.upload_data_to_blob_and_queue(data_to_upload, blob_name, @database_name, @table_name,
|
|
227
|
-
compression_enabled)
|
|
229
|
+
compression_enabled, @ingestion_mapping_reference)
|
|
228
230
|
if @shutdown_called || !@delayed
|
|
229
231
|
commit_write(chunk.unique_id)
|
|
230
232
|
if @shutdown_called
|