fluent-plugin-clickhouse-json-format-output 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f72bb8f8db72bc05e7d66e40b4b2b8d804fc55141eb5de9311d56e357b112b2b
+  data.tar.gz: e95b49aeb219ca98f5be627f19cc555007fd57739e227d6e4a8ee8fc45ae8020
+SHA512:
+  metadata.gz: cc9b3fd21641a97a1759667af5ae965c91fda0d68a4dbea2483b1a2872fcd9c38d4d810e3512bbc2ee5088d30c50c0e7012ea1c83cadae19c129c137b8473b18
+  data.tar.gz: 508b8d9a5571208b42ed4f2dde5d62655c89cd4fc7672504ff23b17661a44ca19e21537b427f11607c44ffab540a0472797bc9ab3564917cdc2fe9cf257cafe0

data/lib/fluent/plugin/out_clickhousejson.rb

@@ -0,0 +1,80 @@
+# -*- coding: UTF-8 -*-
+
+module Fluent
+  class ClickHouseJsonOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('clickhousejson', self)
+
+    include Fluent::SetTimeKeyMixin
+    include Fluent::SetTagKeyMixin
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :string, :default => 8123
+    config_param :url, :string, :default => nil
+    config_param :database, :string, :default => 'default'
+    config_param :username, :string, :default => 'default'
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :table, :string, :default => nil
+
+    attr_accessor :handler
+    attr_accessor :path
+
+    def initialize
+      super
+
+      require 'click_house'
+    end
+
+    unless method_defined?(:log)
+      define_method('log') { $log }
+    end
+
+    def configure(conf)
+      super
+      @cf = {}
+      if conf['username']
+        @cf['username'] = conf['username']
+      end
+      if conf['password']
+        @cf['password'] = conf['password']
+      end
+
+      if conf['url']
+        @url = @url.split(',').map { |m| m.strip }
+      elsif conf['host']
+        @cf['host'] = conf['host']
+      end
+      if conf['port']
+        @cf['port'] = conf['port']
+      end
+
+      @cf['database'] = conf['database']
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def format(tag, time, record)
+       [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      ClickHouse.config do |config|
+        config.host = @cf['host']
+        config.port = @cf['port']
+        config.username = @cf['username']
+        config.password = @cf['password']
+      end
+      ClickHouse.connection.insert(@table) do |rows|
+        chunk.msgpack_each { |tag, time, record|
+          rows << JSON.parse(JSON[record])
+        }
+        rows
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-clickhouse-json-format-output
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- wangchao
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2022-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: click_house
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: Fluentd output inserted into ClickHouse  with json format as fast column-oriented
+  OLAP DBMS.
+email:
+- 1695040842@qq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fluent/plugin/out_clickhousejson.rb
+homepage: https://github.com/wangchaoforever
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin for inserting into ClickHouse with json format.
+test_files: []