fluent-plugin-timestream 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 548e43a61eddcb48d440e888d4708422698da78b5584a7c52d4185aee5aa5c07
+  data.tar.gz: eddbfba2ca3458b060431ef6bc38ecdbeda4bebf4800f84577d824108c7107eb
+SHA512:
+  metadata.gz: d8da4123c8e56eaa782a98246002708582d2120acb5a2f9d012d0d4a8624b275d13671868765e4276dfea3e323c90d1b3a6f65ed57e2f23b17845b269770449d
+  data.tar.gz: ce70a33c5502a452b3b20db2543e8f1dd02e684f68353173405dfc50f7dbed65c108ca5d96c352df770167b1304d7cdc2ea7c028ba7825031045eb0ce078539c

data/.circleci/config.yml

@@ -0,0 +1,42 @@
+version: 2.1
+orbs:
+  ruby: circleci/ruby@0.1.2 
+
+jobs:
+  build_and_test:
+    parameters:
+      ruby-version:
+        type: string
+    docker:
+      - image: circleci/ruby:<< parameters.ruby-version >>-buster
+    executor: ruby/default
+    steps:
+      - checkout
+      - run:
+          name: Execution environments
+          command: echo "Running tests with ruby $(ruby -v), bundler $(bundle -v)"
+      - restore_cache:
+          name: Restore installed gems
+          keys:
+            - v1-fluent-timestream-<< parameters.ruby-version >>
+      - run:
+          name: Bundle install
+          command: bundle install
+      - save_cache:
+          name:  Save installed gems
+          paths:
+            - /usr/local/bundle/bin
+            - /usr/local/bundle/extensions
+            - /usr/local/bundle/gems
+            - /usr/local/bundle/specifications
+          key: v1-fluent-timestream-<< parameters.ruby-version >>-{{ checksum "Gemfile.lock" }}
+      - run:
+          name: Test
+          command: rake test
+workflows:
+  build_and_test:
+    jobs:
+      - build_and_test:
+          matrix:
+            parameters:
+              ruby-version: ["2.6.3", "2.6.5", "2.6", "2.7", "3"]

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+td-agent.conf
+fluent.conf

data/.rubocop.yml

@@ -0,0 +1,41 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Style/FrozenStringLiteralComment:
+  Enabled: true
+
+Style/RedundantReturn:
+  Enabled: false
+
+Style/NegatedIf:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Layout/EmptyLinesAroundModuleBody:
+  Enabled: false
+
+Layout/IndentationConsistency:
+   EnforcedStyle: indented_internal_methods
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'test/*.rb'
+    - 'test/**/*.rb'
+    - 'test/**/**/*.rb'
+
+Layout/LineLength:
+  Max: 90
+
+Style/AndOr:
+  EnforcedStyle: conditionals
+
+Layout/EmptyLineAfterGuardClause:
+  Enabled: false

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-timestream.gemspec
+gemspec
+
+gem 'rake', '~> 13.0'
+
+gem 'rubocop', '~> 1.7'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Studist Corporation
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+# fluent-plugin-timestream
+[![Status](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream.svg?style=shield)](https://circleci.com/gh/StudistCorporation/fluent-plugin-timestream)
+
+Fluentd output plugin for Amazon Timestream.
+
+
+## Installation
+You can install it as follows:
+
+    $ fluent-gem install fluent-plugin-timestream
+
+## Configuration
+Please refer to the [sample config file](https://github.com/StudistCorporation/fluent-plugin-timestream/blob/main/fluent.conf.sample)
+
+## Note
+The plugin converts `null` values in the log to empty string.  
+e.g. `{key_name: null}` => `{key_name: ""}`  
+  
+When writing Timestream records, `TimeUnit` is always set to `SECONDS`  
+  
+Configuring multiple `MeasureName`s is not supported.

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task default: [:build]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'fluent/plugin/timestream'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-timestream.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'lib/fluent/plugin/timestream/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-timestream'
+  spec.version       = Fluent::Plugin::Timestream::VERSION
+  spec.authors       = ['Studist Corporation']
+
+  spec.summary       = 'Fluentd output plugin which writes Amazon Timestream record.'
+  spec.license       = 'MIT'
+
+  spec.metadata['source_code_uri'] = 'https://github.com/StudistCorporation/fluent-plugin-timestream'
+  spec.metadata['changelog_uri'] = 'https://github.com/StudistCorporation/fluent-plugin-timestream/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.required_ruby_version = ">= 2.6.3"
+  spec.add_dependency "aws-sdk-timestreamwrite", ">=1.2.0"
+  spec.add_dependency "fluentd", ">= 1.11.0"
+
+  spec.add_development_dependency "net-empty_port", ">= 0.0.2"
+  spec.add_development_dependency "test-unit", ">= 3.3.9"
+  spec.add_development_dependency "webrick", ">= 1.4.2"
+end

data/fluent.conf.sample

@@ -0,0 +1,45 @@
+<source>
+  @type tail
+  format ltsv
+  path /path/to/sample.log
+  pos_file /tmp/sample.pos
+  tag "sample.log"
+</source>
+
+<match sample.log>
+  @type timestream
+
+  # Amazon Timestream database name.
+  # If not specified, the AWS_TIMESTREAM_DATABASE environment variable will be used instead
+  database "sampleDB"
+
+  # Amazon Timestream table name.
+  # If not specified, the AWS_TIMESTREAM_TABLE environment variable will be used instead
+  table "sampleTable"
+
+  # AWS access key id, secret key, region.
+  # If not specified, search it from environment variable or aws config file.
+  # For more details see https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/TimestreamWrite/Client.html
+  aws_key_id "XXXXXXXX"
+  aws_sec_key "XXXXXXXX"
+  region "us-east-1"
+
+  # Specify which key should be 'measure'.
+  # If not, plugin sends dummy measure and writes all keys and values as dimensions.
+  # Dummy measure is as follows:
+  #   MeasureName: '-'
+  #   MeasureValue: '-'
+  #   MeasureValueType: 'VARCHAR'
+  #<measure>
+  #  name "measureNameXXX"
+  #  type "VARCHAR"
+  #</measure>
+
+  # 'chunk_limit_records' must be configured less or equal to 100.
+  # If not, plugin may fails to write record.
+  # For now, plugin sends records in buffer all at once.
+  # However Amazon Timestream currently accepts less or equal to 100 records.
+  <buffer>
+    chunk_limit_records 100
+  </buffer>
+</match>

data/lib/fluent/plugin/out_timestream.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-core'
+require 'aws-sdk-timestreamwrite'
+require_relative 'timestream/version'
+
+module Fluent
+  module Plugin
+    # Fluent plugin for Amazon Timestream
+    class TimestreamOutput < Fluent::Plugin::Output
+      Fluent::Plugin.register_output('timestream', self)
+
+      config_param :region, :string, default: nil
+
+      config_param :aws_key_id, :string, secret: true, default: nil
+      config_param :aws_sec_key, :string, secret: true, default: nil
+
+      config_param :database, :string, default: nil
+      config_param :table, :string, default: nil
+      config_section :measure,
+                     param_name: 'target_measure', required: false, multi: false do
+        config_param :name, :string
+        config_param :type, :string
+      end
+
+      config_param :endpoint, :string, default: nil
+      config_param :ssl_verify_peer, :bool, default: true
+
+      def configure(conf)
+        super
+        options = credential_options
+        options[:region] = @region if @region
+        options[:endpoint] = @endpoint if @endpoint
+        options[:ssl_verify_peer] = @ssl_verify_peer
+        @client = Aws::TimestreamWrite::Client.new(options)
+
+        @database = ENV['AWS_TIMESTREAM_DATABASE'] if @database.nil?
+        @table = ENV['AWS_TIMESTREAM_TABLE'] if @table.nil?
+      end
+
+      def credential_options
+        if @aws_key_id && @aws_sec_key
+          {
+            access_key_id: @aws_key_id,
+            secret_access_key: @aws_sec_key
+          }
+        else
+          {}
+        end
+      end
+
+      def formatted_to_msgpack_binary
+        true
+      end
+
+      def format(_tag, time, record)
+        [time, record].to_msgpack
+      end
+
+      def create_timestream_record(dimensions, time, measure)
+        measure = { name: '-', value: '-', type: 'VARCHAR' } if measure.empty?
+        {
+          dimensions: dimensions,
+          time: time.to_s,
+          time_unit: 'SECONDS',
+          measure_name: measure[:name],
+          measure_value: measure[:value].to_s,
+          measure_value_type: measure[:type]
+        }
+      end
+
+      def create_timestream_dimension(key, value)
+        {
+          dimension_value_type: 'VARCHAR',
+          name: key,
+          value: value.to_s
+        }
+      end
+
+      def create_timestream_dimensions_and_measure(record)
+        dimensions = []
+        measure = {}
+        record.each do |k, v|
+          if @target_measure && k == @target_measure[:name]
+            measure = { name: k, value: v, type: @target_measure[:type] }
+            next
+          end
+          dimensions.push(create_timestream_dimension(k, v))
+        end
+        return [dimensions, measure]
+      end
+
+      def create_timestream_records(chunk)
+        timestream_records = []
+        chunk.each do |time, record|
+          dimensions, measure = create_timestream_dimensions_and_measure(record)
+          timestream_records.push(create_timestream_record(dimensions, time, measure))
+        end
+
+        log.info("read #{timestream_records.length} records from chunk")
+        timestream_records
+      end
+
+      def write(chunk)
+        @client.write_records(
+          database_name: @database,
+          table_name: @table,
+          records: create_timestream_records(chunk)
+        )
+      rescue Aws::TimestreamWrite::Errors::RejectedRecordsException => e
+        log.error(e.rejected_records)
+      rescue StandardError => e
+        log.error(e.message)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/timestream/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module Timestream
+      VERSION = '0.1.0'
+    end
+  end
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-timestream
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Studist Corporation
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-timestreamwrite
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.11.0
+- !ruby/object:Gem::Dependency
+  name: net-empty_port
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.9
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.9
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.4.2
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- fluent-plugin-timestream.gemspec
+- fluent.conf.sample
+- lib/fluent/plugin/out_timestream.rb
+- lib/fluent/plugin/timestream/version.rb
+homepage: 
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/StudistCorporation/fluent-plugin-timestream
+  changelog_uri: https://github.com/StudistCorporation/fluent-plugin-timestream/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin which writes Amazon Timestream record.
+test_files: []