fluent-plugin-griddb 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dfa1f367763ceb308d6cac0dfc57951bbc81e7ad46a72b6e81feece042474a6c
+  data.tar.gz: 0f94a3e33a970b76beddcd1d387406556e24c11761611d5352e24aa5d98ea55a
+SHA512:
+  metadata.gz: 605222fa8fb0f4b5a38f06f10867586674f7a53bb9866557482c1b0760e986a2487837e55a22ebc055734a200ea1c32255f32e7d052fd9fec55d4c33ef3dc7d9
+  data.tar.gz: 67bd8d6eeedb6a8dbbe2749cb19b001cd6838123a0cb0668c7a3230ee1548f3d0f62b3e88de4664c2594a7e4c6c21fa20be97e7e4dbbd478dc76f8b3893186f9

data/lib/fluent/plugin/griddb_auth.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.dirname(__FILE__)
+
+module Fluent
+	module GridDBAuthParams
+		def self.included(klass)
+		  klass.instance_eval {
+			desc "Host to GridDB Server"
+			# Endpoint URL ex. http://localhost.local/api/
+			config_param :host, :string
+			
+			desc "Cluster name"
+			# Cluster name to GridDB Server
+			config_param :cluster, :string
+			
+			desc "Database name"
+			# Database name in GridDB Server
+			config_param :database, :string
+			
+			desc "Container name"
+			# Container name in database
+			config_param :container, :string
+		   
+			desc "Username of GridDB account"
+			# GridDB username account
+			config_param :username, :string, :default => ''
+
+			desc "Password of GridDB account"
+			# GridDB password account
+			config_param :password, :string, :default => '', :secret => true
+	  
+			# Switch non-buffered/buffered plugin
+			config_param :buffered, :bool, :default => false
+		  }
+		end
+	end
+  
+  module GridDBAuth
+
+    # Set type for request body
+    def set_body(req, record)
+       req.body = Yajl.dump(record)
+       req['Content-Type'] = 'application/json'
+    end
+
+    # Create new request
+    def create_request(record)
+      url = format_url()
+      uri = URI.parse(URI.encode(url))
+      req = Net::HTTP::Put.new(uri.request_uri)
+	  req.basic_auth(@username, @password)
+      set_body(req, record)
+      return req, uri
+    end
+
+    # Send request
+    def send_request(req, uri)
+      res = nil
+      begin      
+        @last_request_time = Time.now.to_f        
+        res = Net::HTTP.start(uri.host, uri.port) {|http| http.request(req) }
+  
+      rescue => e 
+        log.warn "Net::HTTP.#{req.method.capitalize} raises exception: #{e.class}, '#{e.message}'"
+        raise e if @raise_on_error
+      else
+         unless res and res.is_a?(Net::HTTPSuccess)
+            res_summary = if res
+                             "#{res.code} #{res.message} #{res.body}"
+                          else
+                             "res=nil"
+                          end
+            log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+         end
+      end
+    end
+    
+    # Get URL to API
+    def format_url()
+		if $real_container.nil? || $real_container.empty?
+			@host + "/griddb/v2/" + @cluster + "/dbs/" + @database + "/containers/" + @container + "/rows"
+		else
+			@host + "/griddb/v2/" + @cluster + "/dbs/" + @database + "/containers/" + $real_container + "/rows"
+		end        
+    end
+
+  end
+end

data/lib/fluent/plugin/griddb_data.rb

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+$: .unshift File.dirname(__FILE__)
+
+module Fluent
+	module GridDBData
+		# Format for chunk data
+		def chunk_convert(chunk)
+			arrays_2 = Array.new
+			tmp = Array.new
+			chunk.msgpack_each do |time, record|
+				tmp = hash_to_array(record)
+				arrays_2.push(tmp)
+			end			
+			arrays_2
+		end
+	
+		#Format for each record
+		def convert(record)
+			arrays_2 = Array.new
+			tmp = hash_to_array(record)
+			arrays_2.push(tmp)
+		end
+
+		# Covert from hash object to array
+		def hash_to_array(record)
+			arr = Array.new
+			record.map do |key, val |
+				arr.push(val)
+			end			
+			arr
+		end
+
+	end
+end

data/lib/fluent/plugin/out_griddb.rb

@@ -0,0 +1,119 @@
+require "fluent/plugin/output"
+require_relative 'griddb_auth'
+require_relative 'griddb_data'
+require "net/http"
+require "uri"
+require "yajl"
+require "date"
+
+module Fluent
+  module Plugin
+    class GriddbOutput < Fluent::Plugin::Output
+      Fluent::Plugin.register_output("griddb", self)
+	  helpers :compat_parameters
+
+      include Fluent::GridDBAuthParams
+      include Fluent::GridDBAuth
+	  include Fluent::GridDBData
+      
+      DEFAULT_BUFFER_TYPE = "memory"
+      DEFAULT_FORMATTER = "json"
+    
+      def initialize
+        super
+      end
+     
+      config_section :buffer do
+        config_set_default :@type, DEFAULT_BUFFER_TYPE
+        config_set_default :chunk_keys, ['tag']
+      end
+
+      config_section :format do
+        config_set_default :@type, DEFAULT_FORMATTER
+      end
+    
+      def configure(conf)
+        compat_parameters_convert(conf, :buffer, :formatter)
+        super
+		
+		if @host.nil? || @host.empty?
+			raise Fluent::ConfigError, "Host must be not null or empty"
+		end
+		
+		if @cluster.nil? || @cluster.empty?
+			raise Fluent::ConfigError, "Cluster must be not null or empty"
+		end
+		
+		if @database.nil? || @database.empty?
+			raise Fluent::ConfigError, "Database must be not null or empty"
+		end
+		
+		if @container.nil? || @container.empty?
+			raise Fluent::ConfigError, "Container must be not null or empty"
+		end
+		
+		if @username.nil? || @username.empty?
+			raise Fluent::ConfigError, "Username must be not null or empty"
+		end
+		
+		if @password.nil? || @password.empty?
+			raise Fluent::ConfigError, "Password must be not null or empty"
+		end
+	
+		raise Fluent::ConfigError, "'tag' in chunk_keys is required." if !@chunk_key_tag && @buffered	
+		
+        if @formatter_config = conf.elements('format').first
+          log.warn "griddb out plugin is not support format section"
+        end
+      end
+
+      def start
+        super
+      end
+    
+      def shutdown
+        super
+      end
+    
+      def prefer_buffered_processing
+        @buffered
+      end
+
+	  # Format data is auto supported by Fluentd
+      def format(tag, time, record)
+		[time, record].to_msgpack
+      end
+    
+      def formatted_to_msgpack_binary?
+        true
+      end
+    
+      def multi_workers_ready?
+        true
+      end
+
+	  # Put record to server
+	  def handle_record(tag, time, record)
+        req, uri = create_request(record)
+        send_request(req, uri)
+      end
+	  
+	  # Overwrite when Not use buffer
+      def process(tag, es)
+        es.each do |time, record|		 
+		  arrays_2 = convert(record)
+          handle_record(tag, time, arrays_2)
+        end
+      end
+    
+	  # Overwrite when use buffer
+      def write(chunk)
+        tag = chunk.metadata.tag		
+        $real_container = extract_placeholders(@container, chunk)
+		arrays_2 = chunk_convert(chunk)
+		handle_record(tag, DateTime.now, arrays_2)
+      end
+     
+    end
+  end
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-griddb
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- TOSHIBA Digital Solutions Corporation
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-11-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Put data to GridDB server via Put row API
+email:
+- contact@griddb.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/griddb_auth.rb
+- lib/fluent/plugin/griddb_data.rb
+- lib/fluent/plugin/out_griddb.rb
+homepage: https://github.com/griddb/fluent-plugin-griddb
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key:
+specification_version: 4
+summary: Put data to GridDB
+test_files: []