fluent-plugin-timestream 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7666d682c4ca710160fb52118b24b10f189b763579fac67b5593f2e0492bdb49
-  data.tar.gz: '0947bacb4d5410a5c3246f1c1124fa5638e9b93aadabdb6375f21175097a7cc1'
+  metadata.gz: 39a7dfff0e1443346bc334799685c2faed6e546aa0ad4a8782392e6368ca9c19
+  data.tar.gz: dd333c9f9218c7582df76efe842b346bea39561f0ff23cbde73a83f94bc33a26
 SHA512:
-  metadata.gz: aa9828797c6851f7a6579f4a714fd5569a646fe1d372291ea48e6c6b62c3b9ae945c79ad7870088d7ead2dca39840e0ef405a09477354ba5abaada2aee33593b
-  data.tar.gz: be95ba6868b25490a2d071356a08b24960ce831bfc413806a3cb488488d339ea4b09c0681c58573ab314457865e675eb7083d97202909c15bc55ba838f194b26
+  metadata.gz: 57a1df5e1b0e8fe4bb2a3054df8946cdaa11a2ccdb0c2be117a85294dd7e62bb1720b9c9684bbbec8a486090cc8274a661b2f9742d61d7f4ea4fdde008612291
+  data.tar.gz: 0d710f85903d023fddbd5c3a540e6f05477c17e5a8ac0418af199dd0aa7caca27e8aa587d9fa2f246002bab410abc8bd25c811fd132700184834bc2d01051daf

data/README.md

@@ -24,5 +24,3 @@ e.g. `{dimension1: null, dimension2: "", measure: "value"}` => ignores this reco
 
 The plugin ignores record when measure specified in the config has `null` or empty value.  
 e.g. `{dimension1: "value", measure: ""}` => ignores this record
-
-Configuring multiple `MeasureName`s is not supported.

data/fluent.conf.sample

@@ -65,10 +65,30 @@
   #   MeasureName: '-'
   #   MeasureValue: '-'
   #   MeasureValueType: 'VARCHAR'
+  #
+  # Single measure example:
   #<measure>
   #  name "measureNameXXX"
   #  type "VARCHAR"
   #</measure>
+  #
+  # To specify multiple measures, the parent measure must have type "MULTI" and contain nested measure definitions.
+  # For more details about multi-measure records in Timestream, see:
+  # https://docs.aws.amazon.com/timestream/latest/developerguide/writes.html
+  #
+  # Multi-measure example:
+  #<measure>
+  #  name "multiMeasureNameXXX"
+  #  type "MULTI"
+  #  <measure>
+  #    name "measureName1XXX"
+  #    type "VARCHAR"
+  #  </measure>
+  #  <measure>
+  #    name "measureName2XXX"
+  #    type "BIGINT"
+  #  </measure>
+  #</measure>
 
   # 'chunk_limit_records' must be configured less or equal to 100.
   # If not, plugin may fails to write record.

data/lib/fluent/plugin/out_timestream.rb

@@ -18,6 +18,8 @@ module Fluent
           NANOSECONDS
         ].freeze
 
+      DUMMY_MEASURE = { name: '-', value: '-', type: 'VARCHAR' }.freeze
+
       # Raise when measure has empty value
       class EmptyValueError < StandardError
         def initialize(key_name = '')
@@ -45,6 +47,11 @@ module Fluent
                      param_name: 'target_measure', required: false, multi: false do
         config_param :name, :string
         config_param :type, :string
+        config_section :measure,
+                       param_name: 'multi_measures', required: false, multi: true do
+          config_param :name, :string
+          config_param :type, :string
+        end
       end
       config_param :time_unit, :string, default: 'SECONDS'
       config_param :time_key, default: nil
@@ -89,16 +96,13 @@ module Fluent
         [time, record].to_msgpack
       end
 
-      def create_timestream_record(dimensions, time, measure)
+      def create_timestream_record(dimensions, time, measures)
         raise NoDimensionsError if dimensions.empty?
-        measure = { name: '-', value: '-', type: 'VARCHAR' } if measure.empty?
         {
           dimensions: dimensions,
           time: time.to_s,
           time_unit: @time_unit,
-          measure_name: measure[:name],
-          measure_value: measure[:value],
-          measure_value_type: measure[:type]
+          **build_measure_payload(measures)
         }
       end
 
@@ -116,7 +120,7 @@ module Fluent
         }
       end
 
-      def create_timestream_measure(key, value)
+      def create_timestream_measure(key, value, type)
         value = value.to_s
 
         # Timestream does not accept empty string.
@@ -126,21 +130,35 @@ module Fluent
         {
           name: key,
           value: value,
-          type: @target_measure[:type]
+          type: type
         }
       end
 
-      def create_timestream_dimensions_and_measure(record)
-        measure = {}
-        dimensions = record.each_with_object([]) do |(k, v), result|
-          if @target_measure && k == @target_measure[:name]
-            measure = create_timestream_measure(k, v)
-            next
+      def create_timestream_dimensions_and_measures(record)
+        record.each_with_object([[], []]) do |(key, value), (dimensions, measures)|
+          measure_type = measure_types[key]
+          if measure_type
+            measure = create_timestream_measure(key, value, measure_type)
+            measures << measure if measure
+          else
+            dimension = create_timestream_dimension(key, value)
+            dimensions << dimension if dimension
           end
-          dimension = create_timestream_dimension(k, v)
-          result.push(dimension) unless dimension.nil?
         end
-        return [dimensions, measure]
+      end
+
+      def measure_types
+        @measure_types ||= if @target_measure.nil?
+                             {}
+                           elsif multi_measure?
+                             @target_measure.multi_measures.to_h { |m| [m.name, m.type] }
+                           else
+                             { @target_measure.name => @target_measure.type }
+                           end
+      end
+
+      def multi_measure?
+        @target_measure&.type == 'MULTI'
       end
 
       # rubocop:disable Metrics/MethodLength
@@ -148,8 +166,8 @@ module Fluent
         timestream_records = []
         chunk.each do |time, record|
           time = record.delete(@time_key) unless @time_key.nil?
-          dimensions, measure = create_timestream_dimensions_and_measure(record)
-          timestream_records.push(create_timestream_record(dimensions, time, measure))
+          dimensions, measures = create_timestream_dimensions_and_measures(record)
+          timestream_records.push(create_timestream_record(dimensions, time, measures))
         rescue EmptyValueError, NoDimensionsError => e
           log.warn("ignored record due to (#{e})")
           log.debug("ignored record details: #{record}")
@@ -177,6 +195,31 @@ module Fluent
         log.error(e.rejected_records)
       end
 
+      def build_measure_payload(measures)
+        if multi_measure? && !measures.empty?
+          multi_measure_payload(measures)
+        else
+          single_measure_payload(measures)
+        end
+      end
+
+      def multi_measure_payload(measures)
+        {
+          measure_name: @target_measure.name,
+          measure_value_type: 'MULTI',
+          measure_values: measures
+        }
+      end
+
+      def single_measure_payload(measures)
+        measure = measures.empty? ? DUMMY_MEASURE : measures.first
+        {
+          measure_name: measure[:name],
+          measure_value: measure[:value],
+          measure_value_type: measure[:type]
+        }
+      end
+
     end
     # rubocop: enable Metrics/ClassLength
   end

data/lib/fluent/plugin/timestream/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module Timestream
-      VERSION = '1.0.0'
+      VERSION = '1.1.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-timestream
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Studist Corporation
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-13 00:00:00.000000000 Z
+date: 2024-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-timestreamwrite
@@ -80,8 +80,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.2
-description: 
-email: 
+description:
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -99,13 +99,13 @@ files:
 - fluent.conf.sample
 - lib/fluent/plugin/out_timestream.rb
 - lib/fluent/plugin/timestream/version.rb
-homepage: 
+homepage:
 licenses:
 - MIT
 metadata:
   source_code_uri: https://github.com/StudistCorporation/fluent-plugin-timestream
   changelog_uri: https://github.com/StudistCorporation/fluent-plugin-timestream/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -120,8 +120,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: Fluentd output plugin which writes Amazon Timestream record.
 test_files: []