logstash-output-clickhouse 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1fbf922d44fe511cc743b7fd7c86ad3ea3de2ed8ed8afc3bbe549eb7dace1b03
+  data.tar.gz: 6ead7854b8ef425526d9617a3e178c9ff7298eb57fac8f6a84b04c75e9873f08
+SHA512:
+  metadata.gz: c5f5f85f5dc3e2f188cac06a78289e269d6b47a114fec1b2defd37fa343ce3af6cf1ace1a85d8fd5e323dd61866ad944ea2b5d45d6cea1f23f1c797997f24007
+  data.tar.gz: 88e06fa018fc126c056cfc4a8c8c6237d5921d10449aa1878e506746f77cddc8b3db88dc73f34f5a631c7fd273ca95274a4fc44d348603514d123307f9150d22

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2017 GetResponse.com
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,43 @@
+# I switched to vector -> https://github.com/timberio/vector.
+
+# Logstash Plugin
+
+This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch). 
+
+It has been modified to support ClickHouse JSON Format, but also supports fault tolerance.
+
+# Usage
+
+Please note that the name of the plugin when used is `clickhouse`, it only supports json in its current form. If further output formats are added in the future, this might change back to json_batch.
+
+    output {
+      clickhouse {
+        headers => ["Authorization", "Basic YWRtaW46cGFzc3dvcmQxMjM="]
+        http_hosts => ["http://your.clickhouse1/", "http://your.clickhouse2/", "http://your.clickhouse3/"]
+        table => "table_name"
+        mutations => {
+          "to1" => "from1"
+          "to2" => [ "from2", "(.)(.)", '\1\2' ]
+        }
+      }
+    }
+
+## Other custom options
+* `save_on_failure` (default: true) - enable / disable request body save on failure
+* `save_dir` (default: /tmp) - directory where failed request body will be saved
+* `automatic_retries` (default: 1) - number of connect retry attempts to each host in `http_hosts`
+* `request_tolerance` (default: 5) - number of http request send retry attempts if response status code is not 200
+* `backoff_time` (default: 3) - time to wait in seconds for next retry attempt of connect or request
+
+Default batch size is 50, with a wait of at most 5 seconds per send. These can be tweaked with the parameters `flush_size` and `idle_flush_time` respectively.
+
+# Installation 
+
+The easiest way to use this plugin is by installing it through rubygems like any other logstash plugin. To get the latest versio installed, you should run the following command: `bin/logstash-plugin install logstash-output-clickhouse`
+
+# Building the gem and installing a local version
+
+To build the gem yourself, use `gem build logstash-output-clickhouse.gemspec` in the root of this repository. Alternatively, you can download a built version of the gem from the `dist` branch of this repository. 
+
+To install, run the following command, assuming the gem is in the local directory: `$LOGSTASH_HOME/bin/plugin install logstash-output-clickhouse-X.Y.Z.gem`
+

data/lib/logstash/outputs/clickhouse.rb

@@ -0,0 +1,280 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/json"
+require "logstash/util/shortname_resolver"
+require "uri"
+require "stud/buffer"
+require "logstash/plugin_mixins/http_client"
+require "securerandom"
+
+
+class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
+  include LogStash::PluginMixins::HttpClient
+  include Stud::Buffer
+
+  concurrency :single
+
+  config_name "clickhouse"
+
+  config :http_hosts, :validate => :array, :required => true
+
+  config :table, :validate => :string, :required => true
+  
+  # Custom headers to use
+  # format is `headers => ["X-My-Header", "%{host}"]`
+  config :headers, :validate => :hash
+
+  config :flush_size, :validate => :number, :default => 50
+
+  config :idle_flush_time, :validate => :number, :default => 5
+
+  config :pool_max, :validate => :number, :default => 50
+
+  config :save_on_failure, :validate => :boolean, :default => true
+
+  config :save_dir, :validate => :string, :default => "/tmp"
+
+  config :save_file, :validate => :string, :default => "failed.json"
+
+  config :request_tolerance, :validate => :number, :default => 5
+  
+  config :backoff_time, :validate => :number, :default => 3
+
+  config :automatic_retries, :validate => :number, :default => 3
+
+  config :mutations, :validate => :hash, :default => {}
+
+  config :host_resolve_ttl_sec, :validate => :number, :default => 120
+
+  def print_plugin_info()
+    @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-clickhouse/ }
+    @plugin_name = @@plugins[0].name
+    @plugin_version = @@plugins[0].version
+    @logger.info("Running #{@plugin_name} version #{@plugin_version}")
+
+    @logger.info("Initialized clickhouse with settings",
+      :flush_size => @flush_size,
+      :idle_flush_time => @idle_flush_time,
+      :request_tokens => @pool_max,
+      :http_hosts => @http_hosts,
+      :http_query => @http_query,
+      :headers => request_headers)
+  end
+
+  def register
+    # Handle this deprecated option. TODO: remove the option
+    #@ssl_certificate_validation = @verify_ssl if @verify_ssl
+
+    # We count outstanding requests with this queue
+    # This queue tracks the requests to create backpressure
+    # When this queue is empty no new requests may be sent,
+    # tokens must be added back by the client on success
+    @request_tokens = SizedQueue.new(@pool_max)
+    @pool_max.times {|t| @request_tokens << true }
+    @requests = Array.new
+    @http_query = "/?query=INSERT%20INTO%20#{table}%20FORMAT%20JSONEachRow"
+
+    @hostnames_pool =
+      parse_http_hosts(http_hosts,
+        ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
+
+    buffer_initialize(
+      :max_items => @flush_size,
+      :max_interval => @idle_flush_time,
+      :logger => @logger
+    )
+
+    print_plugin_info()
+  end # def register
+
+  private
+
+  def parse_http_hosts(hosts, resolver)
+    ip_re = /^[\d]+\.[\d]+\.[\d]+\.[\d]+$/
+
+    lambda {
+      hosts.flat_map { |h|
+        scheme = URI(h).scheme
+        host = URI(h).host
+        port = URI(h).port
+        path = URI(h).path
+
+        if ip_re !~ host
+          resolver.get_addresses(host).map { |ip|
+            "#{scheme}://#{ip}:#{port}#{path}"
+          }
+        else
+          [h]
+        end
+      }
+    }
+  end
+
+  private
+
+  def get_host_addresses()
+    begin
+      @hostnames_pool.call
+    rescue Exception => ex
+      @logger.error('Error while resolving host', :error => ex.to_s)
+    end
+  end
+
+  # This module currently does not support parallel requests as that would circumvent the batching
+  def receive(event)
+    buffer_receive(event)
+  end
+
+  def mutate( src )
+    return src if @mutations.empty?
+    res = {}
+    @mutations.each_pair do |dstkey, source|
+      case source
+        when String then
+          scrkey = source
+          next unless src.key?(scrkey)
+
+          res[dstkey] = src[scrkey]
+        when Array then
+          scrkey = source[0]
+          next unless src.key?(scrkey)
+          pattern = source[1]
+          replace = source[2]
+          res[dstkey] = src[scrkey].sub( Regexp.new(pattern), replace )
+      end
+    end
+    res
+  end
+
+  public
+  def flush(events, close=false)
+    documents = ""  #this is the string of hashes that we push to Fusion as documents
+
+    events.each do |event|
+        documents << LogStash::Json.dump( mutate( event.to_hash() ) ) << "\n"
+    end
+
+    hosts = get_host_addresses()
+
+    make_request(documents, hosts, @http_query, 1, 1, hosts.sample)
+  end
+
+  private
+
+  def save_to_disk(documents)
+    begin
+      file = File.open("#{save_dir}/#{table}_#{save_file}", "a")
+      file.write(documents) 
+    rescue IOError => e
+      log_failure("An error occurred while saving file to disk: #{e}",
+                    :file_name => file_name)
+    ensure
+      file.close unless file.nil?
+    end
+  end
+
+  def delay_attempt(attempt_number, delay)
+    # sleep delay grows roughly as k*x*ln(x) where k is the initial delay set in @backoff_time param
+    attempt = [attempt_number, 1].max
+    timeout = lambda { |x| [delay*x*Math.log(x), 1].max }
+    # using rand() to pick final sleep delay to reduce the risk of getting in sync with other clients writing to the DB
+    sleep_time = rand(timeout.call(attempt)..timeout.call(attempt+1))
+    sleep sleep_time
+  end
+
+  private
+
+  def make_request(documents, hosts, query, con_count = 1, req_count = 1, host = "", uuid = SecureRandom.hex)
+
+    if host == ""
+      host = hosts.pop
+    end
+
+    url = host+query
+
+    # Block waiting for a token
+    #@logger.info("Requesting token ", :tokens => request_tokens.length())
+    token = @request_tokens.pop
+    @logger.debug("Got token", :tokens => @request_tokens.length)
+
+    # Create an async request
+    begin
+      request = client.send(:post, url, :body => documents, :headers => request_headers, :async => true)
+    rescue Exception => e
+      @logger.warn("An error occurred while indexing: #{e.message}")
+    end
+
+    request.on_success do |response|
+      # Make sure we return the token to the pool
+      @request_tokens << token
+
+      if response.code == 200
+        @logger.debug("Successfully submitted", 
+          :size => documents.length,
+          :response_code => response.code,
+          :uuid => uuid)
+      else
+        if req_count >= @request_tolerance
+          log_failure(
+              "Encountered non-200 HTTP code #{response.code}",
+              :response_code => response.code,
+              :url => url,
+              :size => documents.length,
+              :uuid => uuid)
+          if @save_on_failure
+            save_to_disk(documents)
+          end
+        else
+          @logger.info("Retrying request", :url => url, :message => response.message, :response => response.body, :uuid => uuid)
+          delay_attempt(req_count, @backoff_time)
+          make_request(documents, hosts, query, con_count, req_count+1, host, uuid)
+        end
+      end
+    end
+
+    request.on_failure do |exception|
+      # Make sure we return the token to the pool
+      @request_tokens << token
+
+      if hosts.length == 0
+          log_failure("Could not access URL",
+            :url => url,
+            :method => @http_method,
+            :headers => headers,
+            :message => exception.message,
+            :class => exception.class.name,
+            :backtrace => exception.backtrace,
+            :size => documents.length,
+            :uuid => uuid)
+          if @save_on_failure
+            save_to_disk(documents)
+          end
+          return
+      end
+  
+      if con_count >= @automatic_retries
+        host = ""
+        con_count = 0
+      end
+
+      @logger.info("Retrying connection", :url => url, :uuid => uuid)
+      delay_attempt(con_count, @backoff_time)
+      make_request(documents, hosts, query, con_count+1, req_count, host, uuid)
+    end
+
+    client.execute!
+  end
+
+  # This is split into a separate method mostly to help testing
+  def log_failure(message, opts)
+    @logger.error("[HTTP Output Failure] #{message}", opts)
+  end
+
+  def request_headers()
+    headers = @headers || {}
+    headers["Content-Type"] ||= "application/json"
+    headers
+  end
+
+end

data/lib/logstash/util/shortname_resolver.rb

@@ -0,0 +1,40 @@
+require 'resolv'
+require 'mini_cache'
+
+class ShortNameResolver
+  def initialize(ttl:, logger:)
+    @ttl = ttl
+    @store = MiniCache::Store.new
+    @logger = logger
+  end
+
+  private
+  def resolve_cached(shortname)
+    @store.get_or_set(shortname) do
+      addresses = resolve(shortname)
+      raise "Bad shortname '#{shortname}'" if addresses.empty?
+      MiniCache::Data.new(addresses, expires_in: @ttl)
+    end
+  end
+
+  private
+  def resolve(shortname)
+    addresses = Resolv::DNS.open do |dns|
+      dns.getaddresses(shortname).map { |r| r.to_s }
+    end
+
+    @logger.info("Resolved shortname '#{shortname}' to addresses #{addresses}")
+
+    return addresses
+  end
+
+  public
+  def get_address(shortname)
+    return resolve_cached(shortname).sample
+  end
+
+  public
+  def get_addresses(shortname)
+    return resolve_cached(shortname)
+  end
+end

data/logstash-output-clickhouse.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name            = 'logstash-output-clickhouse'
+  s.version         = '0.1.1'
+  s.licenses        = ['Apache-2.0']
+  s.summary         = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
+  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors         = ["kmajk"]
+  s.email           = 'k.majk@getresponse.com'
+  s.homepage        = "http://getresponse.com"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','Gemfile','LICENSE' ]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "<= 7.0.0"
+  s.add_runtime_dependency 'mini_cache', ">= 1.0.0", "< 2.0.0"
+
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-clickhouse
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- kmajk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-08-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-http_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: mini_cache
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: k.majk@getresponse.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/clickhouse.rb
+- lib/logstash/util/shortname_resolver.rb
+- logstash-output-clickhouse.gemspec
+homepage: http://getresponse.com
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: This output lets you `POST` messages as JSON in a batched fashion to ClickHouse
+  HTTP endpoint
+test_files: []