logstash-output-clickhouse 0.1.1 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fbf922d44fe511cc743b7fd7c86ad3ea3de2ed8ed8afc3bbe549eb7dace1b03
-  data.tar.gz: 6ead7854b8ef425526d9617a3e178c9ff7298eb57fac8f6a84b04c75e9873f08
+  metadata.gz: 20cb59bb456f397ff9408209d676deb3c46214abbd874a5499ab5f7387bb80b9
+  data.tar.gz: 471cd0087a01acbcd6ed8efaa93e09f89102a2b2785bcfa7bb117e7ed8dd2b0d
 SHA512:
-  metadata.gz: c5f5f85f5dc3e2f188cac06a78289e269d6b47a114fec1b2defd37fa343ce3af6cf1ace1a85d8fd5e323dd61866ad944ea2b5d45d6cea1f23f1c797997f24007
-  data.tar.gz: 88e06fa018fc126c056cfc4a8c8c6237d5921d10449aa1878e506746f77cddc8b3db88dc73f34f5a631c7fd273ca95274a4fc44d348603514d123307f9150d22
+  metadata.gz: 3319460b03965943afecff0c3b910ebfb57611566f02b255b7fb3afd6b2a7f6781b89990cf667286972ed27bb503b90833a300f20450a0ad3e637c2144b3f37e
+  data.tar.gz: 609792b63cc1aa8d073f272e9539458738914c4ed1a795b22e23d6a86724b6e9c7e0277b533d0e02fa03d40168799197a614d3208e129b8ba534a8bb3ea80aee

data/README.md

@@ -1,6 +1,5 @@
-# I switched to vector -> https://github.com/timberio/vector.
-
 # Logstash Plugin
+[![Gem Version](https://badge.fury.io/rb/logstash-output-clickhouse.svg)](https://badge.fury.io/rb/logstash-output-clickhouse)
 
 This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch). 
 
@@ -19,6 +18,9 @@ Please note that the name of the plugin when used is `clickhouse`, it only suppo
           "to1" => "from1"
           "to2" => [ "from2", "(.)(.)", '\1\2' ]
         }
+        extra_params => {
+          "date_time_input_format" => "best_effort"
+        }
       }
     }
 
@@ -28,6 +30,7 @@ Please note that the name of the plugin when used is `clickhouse`, it only suppo
 * `automatic_retries` (default: 1) - number of connect retry attempts to each host in `http_hosts`
 * `request_tolerance` (default: 5) - number of http request send retry attempts if response status code is not 200
 * `backoff_time` (default: 3) - time to wait in seconds for next retry attempt of connect or request
+* `extra_params` (default: {}) - extra parameters to be passed to the clickhouse http client
 
 Default batch size is 50, with a wait of at most 5 seconds per send. These can be tweaked with the parameters `flush_size` and `idle_flush_time` respectively.
 

data/lib/logstash/outputs/clickhouse.rb

@@ -2,13 +2,11 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
-require "logstash/util/shortname_resolver"
 require "uri"
 require "stud/buffer"
 require "logstash/plugin_mixins/http_client"
 require "securerandom"
 
-
 class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   include LogStash::PluginMixins::HttpClient
   include Stud::Buffer
@@ -20,7 +18,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   config :http_hosts, :validate => :array, :required => true
 
   config :table, :validate => :string, :required => true
-  
+
   # Custom headers to use
   # format is `headers => ["X-My-Header", "%{host}"]`
   config :headers, :validate => :hash
@@ -38,7 +36,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   config :save_file, :validate => :string, :default => "failed.json"
 
   config :request_tolerance, :validate => :number, :default => 5
-  
+
   config :backoff_time, :validate => :number, :default => 3
 
   config :automatic_retries, :validate => :number, :default => 3
@@ -47,19 +45,21 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
 
   config :host_resolve_ttl_sec, :validate => :number, :default => 120
 
+  config :extra_params, :validate => :hash, :default => {}
+
   def print_plugin_info()
-    @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-clickhouse/ }
+    @@plugins = Gem::Specification.find_all { |spec| spec.name =~ /logstash-output-clickhouse/ }
     @plugin_name = @@plugins[0].name
     @plugin_version = @@plugins[0].version
     @logger.info("Running #{@plugin_name} version #{@plugin_version}")
 
     @logger.info("Initialized clickhouse with settings",
-      :flush_size => @flush_size,
-      :idle_flush_time => @idle_flush_time,
-      :request_tokens => @pool_max,
-      :http_hosts => @http_hosts,
-      :http_query => @http_query,
-      :headers => request_headers)
+                 :flush_size => @flush_size,
+                 :idle_flush_time => @idle_flush_time,
+                 :request_tokens => @pool_max,
+                 :http_hosts => @http_hosts,
+                 :http_query => @http_query,
+                 :headers => request_headers)
   end
 
   def register
@@ -71,91 +71,57 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     # When this queue is empty no new requests may be sent,
     # tokens must be added back by the client on success
     @request_tokens = SizedQueue.new(@pool_max)
-    @pool_max.times {|t| @request_tokens << true }
+    @pool_max.times { |t| @request_tokens << true }
     @requests = Array.new
-    @http_query = "/?query=INSERT%20INTO%20#{table}%20FORMAT%20JSONEachRow"
 
-    @hostnames_pool =
-      parse_http_hosts(http_hosts,
-        ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
+    params = { "query" => "INSERT INTO #{table} FORMAT JSONEachRow" }.merge(@extra_params)
+    @http_query = "?#{URI.encode_www_form(params)}"
 
     buffer_initialize(
       :max_items => @flush_size,
       :max_interval => @idle_flush_time,
-      :logger => @logger
+      :logger => @logger,
     )
 
     print_plugin_info()
   end # def register
 
-  private
-
-  def parse_http_hosts(hosts, resolver)
-    ip_re = /^[\d]+\.[\d]+\.[\d]+\.[\d]+$/
-
-    lambda {
-      hosts.flat_map { |h|
-        scheme = URI(h).scheme
-        host = URI(h).host
-        port = URI(h).port
-        path = URI(h).path
-
-        if ip_re !~ host
-          resolver.get_addresses(host).map { |ip|
-            "#{scheme}://#{ip}:#{port}#{path}"
-          }
-        else
-          [h]
-        end
-      }
-    }
-  end
-
-  private
-
-  def get_host_addresses()
-    begin
-      @hostnames_pool.call
-    rescue Exception => ex
-      @logger.error('Error while resolving host', :error => ex.to_s)
-    end
-  end
-
   # This module currently does not support parallel requests as that would circumvent the batching
   def receive(event)
     buffer_receive(event)
   end
 
-  def mutate( src )
+  def mutate(src)
     return src if @mutations.empty?
     res = {}
     @mutations.each_pair do |dstkey, source|
       case source
-        when String then
-          scrkey = source
-          next unless src.key?(scrkey)
-
-          res[dstkey] = src[scrkey]
-        when Array then
-          scrkey = source[0]
-          next unless src.key?(scrkey)
-          pattern = source[1]
-          replace = source[2]
-          res[dstkey] = src[scrkey].sub( Regexp.new(pattern), replace )
+      when String
+        scrkey = source
+        next unless src.key?(scrkey)
+
+        res[dstkey] = src[scrkey]
+      when Array
+        scrkey = source[0]
+        next unless src.key?(scrkey)
+        pattern = source[1]
+        replace = source[2]
+        res[dstkey] = src[scrkey].sub(Regexp.new(pattern), replace)
       end
     end
     res
   end
 
   public
-  def flush(events, close=false)
+
+  def flush(events, close = false)
     documents = ""  #this is the string of hashes that we push to Fusion as documents
 
     events.each do |event|
-        documents << LogStash::Json.dump( mutate( event.to_hash() ) ) << "\n"
+      documents << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
     end
 
-    hosts = get_host_addresses()
+    hosts = @http_hosts.clone
 
     make_request(documents, hosts, @http_query, 1, 1, hosts.sample)
   end
@@ -165,10 +131,10 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   def save_to_disk(documents)
     begin
       file = File.open("#{save_dir}/#{table}_#{save_file}", "a")
-      file.write(documents) 
+      file.write(documents)
     rescue IOError => e
       log_failure("An error occurred while saving file to disk: #{e}",
-                    :file_name => file_name)
+                  :file_name => file_name)
     ensure
       file.close unless file.nil?
     end
@@ -177,21 +143,20 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   def delay_attempt(attempt_number, delay)
     # sleep delay grows roughly as k*x*ln(x) where k is the initial delay set in @backoff_time param
     attempt = [attempt_number, 1].max
-    timeout = lambda { |x| [delay*x*Math.log(x), 1].max }
+    timeout = lambda { |x| [delay * x * Math.log(x), 1].max }
     # using rand() to pick final sleep delay to reduce the risk of getting in sync with other clients writing to the DB
-    sleep_time = rand(timeout.call(attempt)..timeout.call(attempt+1))
+    sleep_time = rand(timeout.call(attempt)..timeout.call(attempt + 1))
     sleep sleep_time
   end
 
   private
 
   def make_request(documents, hosts, query, con_count = 1, req_count = 1, host = "", uuid = SecureRandom.hex)
-
     if host == ""
       host = hosts.pop
     end
 
-    url = host+query
+    url = host + query
 
     # Block waiting for a token
     #@logger.info("Requesting token ", :tokens => request_tokens.length())
@@ -210,25 +175,26 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
       @request_tokens << token
 
       if response.code == 200
-        @logger.debug("Successfully submitted", 
-          :size => documents.length,
-          :response_code => response.code,
-          :uuid => uuid)
+        @logger.debug("Successfully submitted",
+                      :size => documents.length,
+                      :response_code => response.code,
+                      :uuid => uuid)
       else
         if req_count >= @request_tolerance
           log_failure(
-              "Encountered non-200 HTTP code #{response.code}",
-              :response_code => response.code,
-              :url => url,
-              :size => documents.length,
-              :uuid => uuid)
+            "Encountered non-200 HTTP code #{response.code}",
+            :response_code => response.code,
+            :url => url,
+            :size => documents.length,
+            :uuid => uuid,
+          )
           if @save_on_failure
             save_to_disk(documents)
           end
         else
           @logger.info("Retrying request", :url => url, :message => response.message, :response => response.body, :uuid => uuid)
           delay_attempt(req_count, @backoff_time)
-          make_request(documents, hosts, query, con_count, req_count+1, host, uuid)
+          make_request(documents, hosts, query, con_count, req_count + 1, host, uuid)
         end
       end
     end
@@ -238,21 +204,21 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
       @request_tokens << token
 
       if hosts.length == 0
-          log_failure("Could not access URL",
-            :url => url,
-            :method => @http_method,
-            :headers => headers,
-            :message => exception.message,
-            :class => exception.class.name,
-            :backtrace => exception.backtrace,
-            :size => documents.length,
-            :uuid => uuid)
-          if @save_on_failure
-            save_to_disk(documents)
-          end
-          return
+        log_failure("Could not access URL",
+                    :url => url,
+                    :method => @http_method,
+                    :headers => headers,
+                    :message => exception.message,
+                    :class => exception.class.name,
+                    :backtrace => exception.backtrace,
+                    :size => documents.length,
+                    :uuid => uuid)
+        if @save_on_failure
+          save_to_disk(documents)
+        end
+        return
       end
-  
+
       if con_count >= @automatic_retries
         host = ""
         con_count = 0
@@ -260,7 +226,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
 
       @logger.info("Retrying connection", :url => url, :uuid => uuid)
       delay_attempt(con_count, @backoff_time)
-      make_request(documents, hosts, query, con_count+1, req_count, host, uuid)
+      make_request(documents, hosts, query, con_count + 1, req_count, host, uuid)
     end
 
     client.execute!
@@ -276,5 +242,4 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     headers["Content-Type"] ||= "application/json"
     headers
   end
-
 end

data/logstash-output-clickhouse.gemspec

@@ -1,16 +1,16 @@
 Gem::Specification.new do |s|
-  s.name            = 'logstash-output-clickhouse'
-  s.version         = '0.1.1'
-  s.licenses        = ['Apache-2.0']
-  s.summary         = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["kmajk"]
-  s.email           = 'k.majk@getresponse.com'
-  s.homepage        = "http://getresponse.com"
+  s.name = "logstash-output-clickhouse"
+  s.version = "0.1.4"
+  s.licenses = ["Apache-2.0"]
+  s.summary = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["kmajk", "maltoze"]
+  s.email = "atonezzz@gmail.com"
+  s.homepage = "https://github.com/maltoze/logstash-output-clickhouse"
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','Gemfile','LICENSE' ]
+  s.files = Dir["lib/**/*", "spec/**/*", "*.gemspec", "*.md", "Gemfile", "LICENSE"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,10 +20,10 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "<= 7.0.0"
-  s.add_runtime_dependency 'mini_cache', ">= 1.0.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
+  s.add_runtime_dependency "mini_cache", ">= 1.0.0", "< 2.0.0"
 
-  s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'sinatra'
-  s.add_development_dependency 'webrick'
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "sinatra"
+  s.add_development_dependency "webrick"
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-clickhouse
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.4
 platform: ruby
 authors:
 - kmajk
+- maltoze
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2023-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -37,9 +38,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 6.0.0
-    - - "<="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -47,9 +48,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 6.0.0
-    - - "<="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: mini_cache
   requirement: !ruby/object:Gem::Requirement
@@ -115,7 +116,7 @@ dependencies:
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
   a stand-alone program
-email: k.majk@getresponse.com
+email: atonezzz@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -124,9 +125,8 @@ files:
 - LICENSE
 - README.md
 - lib/logstash/outputs/clickhouse.rb
-- lib/logstash/util/shortname_resolver.rb
 - logstash-output-clickhouse.gemspec
-homepage: http://getresponse.com
+homepage: https://github.com/maltoze/logstash-output-clickhouse
 licenses:
 - Apache-2.0
 metadata:

data/lib/logstash/util/shortname_resolver.rb

@@ -1,40 +0,0 @@
-require 'resolv'
-require 'mini_cache'
-
-class ShortNameResolver
-  def initialize(ttl:, logger:)
-    @ttl = ttl
-    @store = MiniCache::Store.new
-    @logger = logger
-  end
-
-  private
-  def resolve_cached(shortname)
-    @store.get_or_set(shortname) do
-      addresses = resolve(shortname)
-      raise "Bad shortname '#{shortname}'" if addresses.empty?
-      MiniCache::Data.new(addresses, expires_in: @ttl)
-    end
-  end
-
-  private
-  def resolve(shortname)
-    addresses = Resolv::DNS.open do |dns|
-      dns.getaddresses(shortname).map { |r| r.to_s }
-    end
-
-    @logger.info("Resolved shortname '#{shortname}' to addresses #{addresses}")
-
-    return addresses
-  end
-
-  public
-  def get_address(shortname)
-    return resolve_cached(shortname).sample
-  end
-
-  public
-  def get_addresses(shortname)
-    return resolve_cached(shortname)
-  end
-end