fluent-plugin-clickhouse-output 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f3b8c77b050b4181339da242a1e39b6bd3019ff0046404395723e57a207e389d
+  data.tar.gz: 674c5dcdc0a8b839482afcd6c646cd5821e0ecd9558ce65aca2f0487eee2824c
+SHA512:
+  metadata.gz: bff6c90bd1af01fd3218e8afe72764c6ded856a1cf99c87e0318ad076759f284a2c4f43a899c85736ea824c8c6f5f474744a99dc530e03d5e182feeaedc6cae7
+  data.tar.gz: 0b540b0d0e2c3d228f8d0a548c436096c92850cc6ebbd11af6daa698db131d107cdcff7299b323db8bc06336a8c5e1e667b9d1388f1b5fef8d1b03feeaaf7514

data/.gitignore

@@ -0,0 +1,2 @@
+/.idea
+*.gem

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,102 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-clickhouse-output (0.0.2)
+      clickhouse (~> 0.1)
+      fluentd (~> 1.12)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.3.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    clickhouse (0.1.10)
+      activesupport (>= 4.1.8)
+      bundler (>= 1.13.4)
+      erubis
+      faraday
+      launchy
+      pond
+      sinatra
+      thor
+    concurrent-ruby (1.1.8)
+    cool.io (1.7.1)
+    erubis (2.7.0)
+    faraday (1.4.2)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    fluentd (1.13.0)
+      bundler
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 1.3.1, < 2.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (>= 1.0, < 3.0)
+      tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    launchy (2.5.0)
+      addressable (~> 2.7)
+    minitest (5.14.4)
+    msgpack (1.4.2)
+    multipart-post (2.1.1)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
+    pond (0.5.0)
+    public_suffix (4.0.6)
+    rack (2.2.3)
+    rack-protection (2.1.0)
+      rack
+    rake (10.5.0)
+    ruby2_keywords (0.0.4)
+    serverengine (2.2.4)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    sinatra (2.1.0)
+      mustermann (~> 1.0)
+      rack (~> 2.2)
+      rack-protection (= 2.1.0)
+      tilt (~> 2.0)
+    strptime (0.2.5)
+    thor (1.1.0)
+    tilt (2.0.10)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.1)
+      tzinfo (>= 1.0.0)
+    webrick (1.7.0)
+    yajl-ruby (1.4.1)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 1.17)
+  fluent-plugin-clickhouse-output!
+  rake (~> 10.0)
+
+BUNDLED WITH
+   1.17.3

data/README.md

@@ -0,0 +1,35 @@
+# Fluent::Plugin::Clickhouse
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fluent-plugin-clickhouse-output'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-clickhouse-output
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/amlivedn/fluent-plugin-clickhouse-output. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Fluent::Plugin::Clickhouse project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/fluent-plugin-clickhouse/blob/master/CODE_OF_CONDUCT.md).

data/fluent-plugin-clickhouse-output.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name = 'fluent-plugin-clickhouse-output'
+  spec.version = '0.0.2'
+  spec.authors = ['Alex Malyi']
+  spec.email = ['am@pixelx.pro']
+
+  spec.summary = %q{Fluentd output plugin for inserting into ClickHouse.}
+  spec.description = %q{Fluentd output inserted into ClickHouse as fast column-oriented OLAP DBMS.}
+  spec.homepage = 'https://github.com/amlivedn/fluentd-plugin-clickhouse-output'
+  spec.license = 'MIT'
+
+  spec.rubyforge_project = 'fluent-plugin-clickhouse-output'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.17'
+  spec.add_development_dependency 'rake', '~> 10.0'
+
+  spec.add_runtime_dependency 'fluentd', '~> 1.12'
+  spec.add_runtime_dependency 'clickhouse', '~> 0.1'
+end

data/lib/fluent/plugin/out_clickhouse.rb

@@ -0,0 +1,80 @@
+# -*- encoding: utf-8 -*-
+
+module Fluent
+  class ClickHouseOutput < Fluent::BufferedOutput
+    Fluent::Plugin.register_output('clickhouse-output', self)
+
+    include Fluent::SetTimeKeyMixin
+    include Fluent::SetTagKeyMixin
+
+    config_param :host, :string, :default => 'localhost'
+    config_param :port, :string, :default => 8192
+    config_param :urls, :string, :default => nil
+    config_param :database, :string, :default => 'default'
+    config_param :username, :string, :default => 'default'
+    config_param :password, :string, :default => nil, :secret => true
+
+    config_param :table, :string, :default => nil
+    config_param :columns, :string, :default => nil
+
+    attr_accessor :handler
+    attr_accessor :path
+
+    def initialize
+      super
+
+      require 'clickhouse'
+    end
+
+    unless method_defined?(:log)
+      define_method('log') { $log }
+    end
+
+    def configure(conf)
+      super
+      if conf['columns']
+        @columns = conf['columns'].split(',').map { |m| m.strip }
+      end
+      @config = {}
+      if conf['username']
+        @config['username'] = conf['username']
+      end
+      if conf['password']
+        @config['password'] = conf['password']
+      end
+
+      if conf['urls']
+        @urls = @urls.split(',').map { |m| m.strip }
+      elsif conf['host']
+        @config['host'] = conf['host']
+      end
+      if conf['port']
+        @config['port'] = conf['port']
+      end
+
+      @config['database'] = conf['database']
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      Clickhouse.establish_connection(@config)
+      Clickhouse.connection.insert_rows(@table, :names => @columns) { |rows|
+        chunk.msgpack_each { |tag, time, record|
+          rows << @columns.map { |m| record[m] }
+        }
+        rows
+      }
+    end
+  end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-clickhouse-output
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Alex Malyi
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2021-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: clickhouse
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: Fluentd output inserted into ClickHouse as fast column-oriented OLAP
+  DBMS.
+email:
+- am@pixelx.pro
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- fluent-plugin-clickhouse-output.gemspec
+- lib/fluent/plugin/out_clickhouse.rb
+homepage: https://github.com/amlivedn/fluentd-plugin-clickhouse-output
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin for inserting into ClickHouse.
+test_files: []