fluent-plugin-timestream 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 548e43a61eddcb48d440e888d4708422698da78b5584a7c52d4185aee5aa5c07
-  data.tar.gz: eddbfba2ca3458b060431ef6bc38ecdbeda4bebf4800f84577d824108c7107eb
+  metadata.gz: c6da936981868da70ec9cd3817cc0b386c23b09723bb6b8ec8ccc5f2a8a0140f
+  data.tar.gz: d996e437118d2db8f58ed1e80a5a82412c8f08c666dcf827532bbc40ab588454
 SHA512:
-  metadata.gz: d8da4123c8e56eaa782a98246002708582d2120acb5a2f9d012d0d4a8624b275d13671868765e4276dfea3e323c90d1b3a6f65ed57e2f23b17845b269770449d
-  data.tar.gz: ce70a33c5502a452b3b20db2543e8f1dd02e684f68353173405dfc50f7dbed65c108ca5d96c352df770167b1304d7cdc2ea7c028ba7825031045eb0ce078539c
+  metadata.gz: d9855ee10803e4bcc1e451182e7ab650df68a7c5eaf62a273f5b8107a824ec23831f98746399ffddf525465e673ba522579a37f167ae16847cb7c830ea06d129
+  data.tar.gz: 4c93d1dd0ebe5c46bcf801e4e88a03da873272a1d70ceff454ae1a39f7d2c0d730777a50470e59ff93546d05e06cde298ca07cee5dcdeb840553719610c45bed

data/.rubocop.yml

@@ -1,5 +1,6 @@
 AllCops:
   TargetRubyVersion: 2.6
+  SuggestExtensions: false
 
 Style/FrozenStringLiteralComment:
   Enabled: true
@@ -38,4 +39,4 @@ Style/AndOr:
   EnforcedStyle: conditionals
 
 Layout/EmptyLineAfterGuardClause:
-  Enabled: false
+  Enabled: false

data/README.md

@@ -1,21 +1,28 @@
 # fluent-plugin-timestream
+
 [![Status](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream.svg?style=shield)](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream)
 
 Fluentd output plugin for Amazon Timestream.
 
-
 ## Installation
+
 You can install it as follows:
 
     $ fluent-gem install fluent-plugin-timestream
 
 ## Configuration
+
 Please refer to the [sample config file](https://github.com/StudistCorporation/fluent-plugin-timestream/blob/main/fluent.conf.sample)
 
 ## Note
-The plugin converts `null` values in the log to empty string.  
-e.g. `{key_name: null}` => `{key_name: ""}`  
-  
-When writing Timestream records, `TimeUnit` is always set to `SECONDS`  
-  
+
+The plugin ignores dimension when it has `null` or empty string value.  
+e.g. `{dimension1: null, dimension2: "value", dimension3: ""}` => `{dimension2: "value"}`
+
+The plugin ignores record when it has no dimensions.  
+e.g. `{dimension1: null, dimension2: "", measure: "value"}` => ignores this record
+
+The plugin ignores record when measure specified in the config has `null` or empty value.  
+e.g. `{dimension1: "value", measure: ""}` => ignores this record
+
 Configuring multiple `MeasureName`s is not supported.

data/fluent.conf.sample

@@ -1,11 +1,35 @@
 <source>
   @type tail
-  format ltsv
+  format json
   path /path/to/sample.log
   pos_file /tmp/sample.pos
   tag "sample.log"
+  # set keep_time_key true if need to convert time.
+  # keep_time_key true
 </source>
 
+# If more than seconds accuracy is needed,
+# convert time and configure time_unit and time_key.
+#
+# e.g.)
+# If record is as follows:
+# {"key": "value", "time":1620000000.123456789}
+#
+# convert it to: 
+# when time unit is MILLISECONDS: 1620000000123
+# when time unit is MICROSECONDS: 1620000000123456
+# when time unit is NANOSECONDS:  1620000000123456789
+# <filter sample.log>
+  # @type record_transformer
+  # enable_ruby true
+  # <record>
+  #   milliseconds_time ${(record["time"].to_f * 1000).to_i}
+  # </record>
+  # Remove time key or plugin sends it as a dimension.
+  # Especially, if time key name is 'time', the plugin fails to write record because 'time' is reserved keyword.
+  # remove_keys time
+#</filter>
+
 <match sample.log>
   @type timestream
 
@@ -24,6 +48,17 @@
   aws_sec_key "XXXXXXXX"
   region "us-east-1"
 
+  # If more than seconds accuracy is needed,
+  # convert time and configure time_unit and time_key.
+  # time_unit: The granularity of the timestamp unit. 
+  #            This value is used for 'TimeUnit' in Timestream record
+  #            Default value: SECONDS
+  #            Valid values: SECONDS MILLISECONDS MICROSECONDS NANOSECONDS
+  # time_key:  Specify record key which contains integer epoch time corresponding to time_unit
+  #            Plugin uses specified epoch time for 'Time' in Timestream record and does not send it as dimension
+  # time_unit "MILLISECONDS"
+  # time_key "milliseconds_time"
+
   # Specify which key should be 'measure'.
   # If not, plugin sends dummy measure and writes all keys and values as dimensions.
   # Dummy measure is as follows:

data/lib/fluent/plugin/out_timestream.rb

@@ -6,8 +6,32 @@ require_relative 'timestream/version'
 
 module Fluent
   module Plugin
+    # rubocop: disable Metrics/ClassLength
     # Fluent plugin for Amazon Timestream
     class TimestreamOutput < Fluent::Plugin::Output
+
+      VALID_TIME_UNIT =
+        %w[
+          SECONDS
+          MILLISECONDS
+          MICROSECONDS
+          NANOSECONDS
+        ].freeze
+
+      # Raise when measure has empty value
+      class EmptyValueError < StandardError
+        def initialize(key_name = '')
+          super("measure has empty value. key name: #{key_name}")
+        end
+      end
+
+      # Raise when record has no dimensions
+      class NoDimensionsError < StandardError
+        def initialize
+          super('record has no dimensions.')
+        end
+      end
+
       Fluent::Plugin.register_output('timestream', self)
 
       config_param :region, :string, default: nil
@@ -22,6 +46,8 @@ module Fluent
         config_param :name, :string
         config_param :type, :string
       end
+      config_param :time_unit, :string, default: 'SECONDS'
+      config_param :time_key, default: nil
 
       config_param :endpoint, :string, default: nil
       config_param :ssl_verify_peer, :bool, default: true
@@ -36,6 +62,7 @@ module Fluent
 
         @database = ENV['AWS_TIMESTREAM_DATABASE'] if @database.nil?
         @table = ENV['AWS_TIMESTREAM_TABLE'] if @table.nil?
+        validate_time_unit
       end
 
       def credential_options
@@ -49,6 +76,11 @@ module Fluent
         end
       end
 
+      def validate_time_unit
+        return if VALID_TIME_UNIT.include?(@time_unit)
+        raise Fluent::ConfigError, "Invalid time_unit: #{@time_unit}"
+      end
+
       def formatted_to_msgpack_binary
         true
       end
@@ -58,60 +90,96 @@ module Fluent
       end
 
       def create_timestream_record(dimensions, time, measure)
+        raise NoDimensionsError if dimensions.empty?
         measure = { name: '-', value: '-', type: 'VARCHAR' } if measure.empty?
         {
           dimensions: dimensions,
           time: time.to_s,
-          time_unit: 'SECONDS',
+          time_unit: @time_unit,
           measure_name: measure[:name],
-          measure_value: measure[:value].to_s,
+          measure_value: measure[:value],
           measure_value_type: measure[:type]
         }
       end
 
       def create_timestream_dimension(key, value)
+        value = value.to_s
+
+        # Timestream does not accept empty string.
+        # Ignore this dimension.
+        return nil if value.empty?
+
         {
           dimension_value_type: 'VARCHAR',
           name: key,
-          value: value.to_s
+          value: value
+        }
+      end
+
+      def create_timestream_measure(key, value)
+        value = value.to_s
+
+        # Timestream does not accept empty string.
+        # By raising error, ignore entire record.
+        raise EmptyValueError, key if value.empty?
+
+        {
+          name: key,
+          value: value,
+          type: @target_measure[:type]
         }
       end
 
       def create_timestream_dimensions_and_measure(record)
-        dimensions = []
         measure = {}
-        record.each do |k, v|
+        dimensions = record.each_with_object([]) do |(k, v), result|
           if @target_measure && k == @target_measure[:name]
-            measure = { name: k, value: v, type: @target_measure[:type] }
+            measure = create_timestream_measure(k, v)
             next
           end
-          dimensions.push(create_timestream_dimension(k, v))
+          dimension = create_timestream_dimension(k, v)
+          result.push(dimension) unless dimension.nil?
         end
         return [dimensions, measure]
       end
 
+      # rubocop:disable Metrics/MethodLength
       def create_timestream_records(chunk)
         timestream_records = []
         chunk.each do |time, record|
+          time = record.delete(@time_key) unless @time_key.nil?
           dimensions, measure = create_timestream_dimensions_and_measure(record)
           timestream_records.push(create_timestream_record(dimensions, time, measure))
+        rescue EmptyValueError, NoDimensionsError => e
+          log.warn("ignored record due to (#{e})")
+          log.debug("ignored record details: #{record}")
+          next
         end
 
-        log.info("read #{timestream_records.length} records from chunk")
         timestream_records
       end
+      # rubocop:enable Metrics/MethodLength
 
       def write(chunk)
+        records = create_timestream_records(chunk)
+        log.info("read #{records.length} records from chunk")
+        write_records(records)
+      end
+
+      def write_records(records)
+        return if records.empty?
         @client.write_records(
           database_name: @database,
           table_name: @table,
-          records: create_timestream_records(chunk)
+          records: records
         )
       rescue Aws::TimestreamWrite::Errors::RejectedRecordsException => e
         log.error(e.rejected_records)
       rescue StandardError => e
         log.error(e.message)
       end
+
     end
+    # rubocop: enable Metrics/ClassLength
   end
 end

data/lib/fluent/plugin/timestream/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module Timestream
-      VERSION = '0.1.0'
+      VERSION = '0.2.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-timestream
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Studist Corporation
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-19 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-timestreamwrite