fluent-plugin-clickhouse-json-format-output 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. checksums.yaml +7 -0
  2. data/lib/fluent/plugin/out_clickhousejson.rb +80 -0
  3. metadata +101 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: f72bb8f8db72bc05e7d66e40b4b2b8d804fc55141eb5de9311d56e357b112b2b
4
+ data.tar.gz: e95b49aeb219ca98f5be627f19cc555007fd57739e227d6e4a8ee8fc45ae8020
5
+ SHA512:
6
+ metadata.gz: cc9b3fd21641a97a1759667af5ae965c91fda0d68a4dbea2483b1a2872fcd9c38d4d810e3512bbc2ee5088d30c50c0e7012ea1c83cadae19c129c137b8473b18
7
+ data.tar.gz: 508b8d9a5571208b42ed4f2dde5d62655c89cd4fc7672504ff23b17661a44ca19e21537b427f11607c44ffab540a0472797bc9ab3564917cdc2fe9cf257cafe0
data/lib/fluent/plugin/out_clickhousejson.rb ADDED
@@ -0,0 +1,80 @@
1
+ # -*- coding: UTF-8 -*-
2
+
3
+ module Fluent
4
+ class ClickHouseJsonOutput < Fluent::BufferedOutput
5
+ Fluent::Plugin.register_output('clickhousejson', self)
6
+
7
+ include Fluent::SetTimeKeyMixin
8
+ include Fluent::SetTagKeyMixin
9
+
10
+ config_param :host, :string, :default => 'localhost'
11
+ config_param :port, :string, :default => 8123
12
+ config_param :url, :string, :default => nil
13
+ config_param :database, :string, :default => 'default'
14
+ config_param :username, :string, :default => 'default'
15
+ config_param :password, :string, :default => nil, :secret => true
16
+ config_param :table, :string, :default => nil
17
+
18
+ attr_accessor :handler
19
+ attr_accessor :path
20
+
21
+ def initialize
22
+ super
23
+
24
+ require 'click_house'
25
+ end
26
+
27
+ unless method_defined?(:log)
28
+ define_method('log') { $log }
29
+ end
30
+
31
+ def configure(conf)
32
+ super
33
+ @cf = {}
34
+ if conf['username']
35
+ @cf['username'] = conf['username']
36
+ end
37
+ if conf['password']
38
+ @cf['password'] = conf['password']
39
+ end
40
+
41
+ if conf['url']
42
+ @url = @url.split(',').map { |m| m.strip }
43
+ elsif conf['host']
44
+ @cf['host'] = conf['host']
45
+ end
46
+ if conf['port']
47
+ @cf['port'] = conf['port']
48
+ end
49
+
50
+ @cf['database'] = conf['database']
51
+ end
52
+
53
+ def start
54
+ super
55
+ end
56
+
57
+ def shutdown
58
+ super
59
+ end
60
+
61
+ def format(tag, time, record)
62
+ [tag, time, record].to_msgpack
63
+ end
64
+
65
+ def write(chunk)
66
+ ClickHouse.config do |config|
67
+ config.host = @cf['host']
68
+ config.port = @cf['port']
69
+ config.username = @cf['username']
70
+ config.password = @cf['password']
71
+ end
72
+ ClickHouse.connection.insert(@table) do |rows|
73
+ chunk.msgpack_each { |tag, time, record|
74
+ rows << JSON.parse(JSON[record])
75
+ }
76
+ rows
77
+ end
78
+ end
79
+ end
80
+ end
metadata ADDED
@@ -0,0 +1,101 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-clickhouse-json-format-output
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - wangchao
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2022-11-28 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.17'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.17'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fluentd
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.12'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.12'
55
+ - !ruby/object:Gem::Dependency
56
+ name: click_house
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '2.1'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.1'
69
+ description: Fluentd output inserted into ClickHouse with json format as fast column-oriented
70
+ OLAP DBMS.
71
+ email:
72
+ - 1695040842@qq.com
73
+ executables: []
74
+ extensions: []
75
+ extra_rdoc_files: []
76
+ files:
77
+ - lib/fluent/plugin/out_clickhousejson.rb
78
+ homepage: https://github.com/wangchaoforever
79
+ licenses:
80
+ - MIT
81
+ metadata: {}
82
+ post_install_message:
83
+ rdoc_options: []
84
+ require_paths:
85
+ - lib
86
+ required_ruby_version: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - ">="
89
+ - !ruby/object:Gem::Version
90
+ version: '0'
91
+ required_rubygems_version: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - ">="
94
+ - !ruby/object:Gem::Version
95
+ version: '0'
96
+ requirements: []
97
+ rubygems_version: 3.1.6
98
+ signing_key:
99
+ specification_version: 4
100
+ summary: Fluentd output plugin for inserting into ClickHouse with json format.
101
+ test_files: []