logstash-output-clickhouse 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fbf922d44fe511cc743b7fd7c86ad3ea3de2ed8ed8afc3bbe549eb7dace1b03
-  data.tar.gz: 6ead7854b8ef425526d9617a3e178c9ff7298eb57fac8f6a84b04c75e9873f08
+  metadata.gz: 05f2b2d96799587dc5678cd80a6ef5d0c62a103797a298dfbe86acb1dbe09b89
+  data.tar.gz: ffeb98ddb2e27497ff54e9304180a2a44e88d093ce33755f842409a2101d4153
 SHA512:
-  metadata.gz: c5f5f85f5dc3e2f188cac06a78289e269d6b47a114fec1b2defd37fa343ce3af6cf1ace1a85d8fd5e323dd61866ad944ea2b5d45d6cea1f23f1c797997f24007
-  data.tar.gz: 88e06fa018fc126c056cfc4a8c8c6237d5921d10449aa1878e506746f77cddc8b3db88dc73f34f5a631c7fd273ca95274a4fc44d348603514d123307f9150d22
+  metadata.gz: 1c6d244d35441f46a0b49b79d1f278004e634334349d3949ec8d49e96fb02e31c942d13f8be72e7ddcd9996687acf81f1378b3587cbab6c6a9a9f93eb6634a6a
+  data.tar.gz: 530b73e05f582bf9b9af8a3be95aa1139fd0b21c3a75b411635453fa9c5e4bc538f8fb8477ab2388fad2560570c92a3623d614c97e69f103850ef6e54d45a23a

data/README.md

@@ -1,5 +1,3 @@
-# I switched to vector -> https://github.com/timberio/vector.
-
 # Logstash Plugin
 
 This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch). 
@@ -19,6 +17,9 @@ Please note that the name of the plugin when used is `clickhouse`, it only suppo
           "to1" => "from1"
           "to2" => [ "from2", "(.)(.)", '\1\2' ]
         }
+        extra_params => {
+          "date_time_input_format" => "best_effort"
+        }
       }
     }
 
@@ -28,6 +29,7 @@ Please note that the name of the plugin when used is `clickhouse`, it only suppo
 * `automatic_retries` (default: 1) - number of connect retry attempts to each host in `http_hosts`
 * `request_tolerance` (default: 5) - number of http request send retry attempts if response status code is not 200
 * `backoff_time` (default: 3) - time to wait in seconds for next retry attempt of connect or request
+* `extra_params` (default: {}) - extra parameters to be passed to the clickhouse http client
 
 Default batch size is 50, with a wait of at most 5 seconds per send. These can be tweaked with the parameters `flush_size` and `idle_flush_time` respectively.
 

data/lib/logstash/outputs/clickhouse.rb

@@ -8,7 +8,6 @@ require "stud/buffer"
 require "logstash/plugin_mixins/http_client"
 require "securerandom"
 
-
 class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   include LogStash::PluginMixins::HttpClient
   include Stud::Buffer
@@ -20,7 +19,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   config :http_hosts, :validate => :array, :required => true
 
   config :table, :validate => :string, :required => true
-  
+
   # Custom headers to use
   # format is `headers => ["X-My-Header", "%{host}"]`
   config :headers, :validate => :hash
@@ -38,7 +37,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   config :save_file, :validate => :string, :default => "failed.json"
 
   config :request_tolerance, :validate => :number, :default => 5
-  
+
   config :backoff_time, :validate => :number, :default => 3
 
   config :automatic_retries, :validate => :number, :default => 3
@@ -47,19 +46,21 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
 
   config :host_resolve_ttl_sec, :validate => :number, :default => 120
 
+  config :extra_params, :validate => :hash, :default => {}
+
   def print_plugin_info()
-    @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-clickhouse/ }
+    @@plugins = Gem::Specification.find_all { |spec| spec.name =~ /logstash-output-clickhouse/ }
     @plugin_name = @@plugins[0].name
     @plugin_version = @@plugins[0].version
     @logger.info("Running #{@plugin_name} version #{@plugin_version}")
 
     @logger.info("Initialized clickhouse with settings",
-      :flush_size => @flush_size,
-      :idle_flush_time => @idle_flush_time,
-      :request_tokens => @pool_max,
-      :http_hosts => @http_hosts,
-      :http_query => @http_query,
-      :headers => request_headers)
+                 :flush_size => @flush_size,
+                 :idle_flush_time => @idle_flush_time,
+                 :request_tokens => @pool_max,
+                 :http_hosts => @http_hosts,
+                 :http_query => @http_query,
+                 :headers => request_headers)
   end
 
   def register
@@ -71,18 +72,20 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     # When this queue is empty no new requests may be sent,
     # tokens must be added back by the client on success
     @request_tokens = SizedQueue.new(@pool_max)
-    @pool_max.times {|t| @request_tokens << true }
+    @pool_max.times { |t| @request_tokens << true }
     @requests = Array.new
-    @http_query = "/?query=INSERT%20INTO%20#{table}%20FORMAT%20JSONEachRow"
+
+    params = { "query" => "INSERT INTO #{table} FORMAT JSONEachRow" }.merge(@extra_params)
+    @http_query = "?#{URI.encode_www_form(params)}"
 
     @hostnames_pool =
       parse_http_hosts(http_hosts,
-        ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
+                       ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
 
     buffer_initialize(
       :max_items => @flush_size,
       :max_interval => @idle_flush_time,
-      :logger => @logger
+      :logger => @logger,
     )
 
     print_plugin_info()
@@ -117,7 +120,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     begin
       @hostnames_pool.call
     rescue Exception => ex
-      @logger.error('Error while resolving host', :error => ex.to_s)
+      @logger.error("Error while resolving host", :error => ex.to_s)
     end
   end
 
@@ -126,33 +129,34 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     buffer_receive(event)
   end
 
-  def mutate( src )
+  def mutate(src)
     return src if @mutations.empty?
     res = {}
     @mutations.each_pair do |dstkey, source|
       case source
-        when String then
-          scrkey = source
-          next unless src.key?(scrkey)
-
-          res[dstkey] = src[scrkey]
-        when Array then
-          scrkey = source[0]
-          next unless src.key?(scrkey)
-          pattern = source[1]
-          replace = source[2]
-          res[dstkey] = src[scrkey].sub( Regexp.new(pattern), replace )
+      when String
+        scrkey = source
+        next unless src.key?(scrkey)
+
+        res[dstkey] = src[scrkey]
+      when Array
+        scrkey = source[0]
+        next unless src.key?(scrkey)
+        pattern = source[1]
+        replace = source[2]
+        res[dstkey] = src[scrkey].sub(Regexp.new(pattern), replace)
       end
     end
     res
   end
 
   public
-  def flush(events, close=false)
+
+  def flush(events, close = false)
     documents = ""  #this is the string of hashes that we push to Fusion as documents
 
     events.each do |event|
-        documents << LogStash::Json.dump( mutate( event.to_hash() ) ) << "\n"
+      documents << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
     end
 
     hosts = get_host_addresses()
@@ -165,10 +169,10 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   def save_to_disk(documents)
     begin
       file = File.open("#{save_dir}/#{table}_#{save_file}", "a")
-      file.write(documents) 
+      file.write(documents)
     rescue IOError => e
       log_failure("An error occurred while saving file to disk: #{e}",
-                    :file_name => file_name)
+                  :file_name => file_name)
     ensure
       file.close unless file.nil?
     end
@@ -177,21 +181,20 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   def delay_attempt(attempt_number, delay)
     # sleep delay grows roughly as k*x*ln(x) where k is the initial delay set in @backoff_time param
     attempt = [attempt_number, 1].max
-    timeout = lambda { |x| [delay*x*Math.log(x), 1].max }
+    timeout = lambda { |x| [delay * x * Math.log(x), 1].max }
     # using rand() to pick final sleep delay to reduce the risk of getting in sync with other clients writing to the DB
-    sleep_time = rand(timeout.call(attempt)..timeout.call(attempt+1))
+    sleep_time = rand(timeout.call(attempt)..timeout.call(attempt + 1))
     sleep sleep_time
   end
 
   private
 
   def make_request(documents, hosts, query, con_count = 1, req_count = 1, host = "", uuid = SecureRandom.hex)
-
     if host == ""
       host = hosts.pop
     end
 
-    url = host+query
+    url = host + query
 
     # Block waiting for a token
     #@logger.info("Requesting token ", :tokens => request_tokens.length())
@@ -210,25 +213,26 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
       @request_tokens << token
 
       if response.code == 200
-        @logger.debug("Successfully submitted", 
-          :size => documents.length,
-          :response_code => response.code,
-          :uuid => uuid)
+        @logger.debug("Successfully submitted",
+                      :size => documents.length,
+                      :response_code => response.code,
+                      :uuid => uuid)
       else
         if req_count >= @request_tolerance
           log_failure(
-              "Encountered non-200 HTTP code #{response.code}",
-              :response_code => response.code,
-              :url => url,
-              :size => documents.length,
-              :uuid => uuid)
+            "Encountered non-200 HTTP code #{response.code}",
+            :response_code => response.code,
+            :url => url,
+            :size => documents.length,
+            :uuid => uuid,
+          )
           if @save_on_failure
             save_to_disk(documents)
           end
         else
           @logger.info("Retrying request", :url => url, :message => response.message, :response => response.body, :uuid => uuid)
           delay_attempt(req_count, @backoff_time)
-          make_request(documents, hosts, query, con_count, req_count+1, host, uuid)
+          make_request(documents, hosts, query, con_count, req_count + 1, host, uuid)
         end
       end
     end
@@ -238,21 +242,21 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
       @request_tokens << token
 
       if hosts.length == 0
-          log_failure("Could not access URL",
-            :url => url,
-            :method => @http_method,
-            :headers => headers,
-            :message => exception.message,
-            :class => exception.class.name,
-            :backtrace => exception.backtrace,
-            :size => documents.length,
-            :uuid => uuid)
-          if @save_on_failure
-            save_to_disk(documents)
-          end
-          return
+        log_failure("Could not access URL",
+                    :url => url,
+                    :method => @http_method,
+                    :headers => headers,
+                    :message => exception.message,
+                    :class => exception.class.name,
+                    :backtrace => exception.backtrace,
+                    :size => documents.length,
+                    :uuid => uuid)
+        if @save_on_failure
+          save_to_disk(documents)
+        end
+        return
       end
-  
+
       if con_count >= @automatic_retries
         host = ""
         con_count = 0
@@ -260,7 +264,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
 
       @logger.info("Retrying connection", :url => url, :uuid => uuid)
       delay_attempt(con_count, @backoff_time)
-      make_request(documents, hosts, query, con_count+1, req_count, host, uuid)
+      make_request(documents, hosts, query, con_count + 1, req_count, host, uuid)
     end
 
     client.execute!
@@ -276,5 +280,4 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     headers["Content-Type"] ||= "application/json"
     headers
   end
-
 end

data/logstash-output-clickhouse.gemspec

@@ -1,16 +1,16 @@
 Gem::Specification.new do |s|
-  s.name            = 'logstash-output-clickhouse'
-  s.version         = '0.1.1'
-  s.licenses        = ['Apache-2.0']
-  s.summary         = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
-  s.authors         = ["kmajk"]
-  s.email           = 'k.majk@getresponse.com'
-  s.homepage        = "http://getresponse.com"
+  s.name = "logstash-output-clickhouse"
+  s.version = "0.1.3"
+  s.licenses = ["Apache-2.0"]
+  s.summary = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["kmajk", "maltoze"]
+  s.email = "atonezzz@gmail.com"
+  s.homepage = "https://github.com/maltoze/logstash-output-clickhouse"
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','Gemfile','LICENSE' ]
+  s.files = Dir["lib/**/*", "spec/**/*", "*.gemspec", "*.md", "Gemfile", "LICENSE"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -20,10 +20,10 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "<= 7.0.0"
-  s.add_runtime_dependency 'mini_cache', ">= 1.0.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-mixin-http_client", ">= 6.0.0", "< 8.0.0"
+  s.add_runtime_dependency "mini_cache", ">= 1.0.0", "< 2.0.0"
 
-  s.add_development_dependency 'logstash-devutils'
-  s.add_development_dependency 'sinatra'
-  s.add_development_dependency 'webrick'
+  s.add_development_dependency "logstash-devutils"
+  s.add_development_dependency "sinatra"
+  s.add_development_dependency "webrick"
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-clickhouse
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - kmajk
+- maltoze
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-04 00:00:00.000000000 Z
+date: 2022-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -37,9 +38,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 6.0.0
-    - - "<="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -47,9 +48,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 6.0.0
-    - - "<="
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: mini_cache
   requirement: !ruby/object:Gem::Requirement
@@ -115,7 +116,7 @@ dependencies:
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
   a stand-alone program
-email: k.majk@getresponse.com
+email: atonezzz@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -126,7 +127,7 @@ files:
 - lib/logstash/outputs/clickhouse.rb
 - lib/logstash/util/shortname_resolver.rb
 - logstash-output-clickhouse.gemspec
-homepage: http://getresponse.com
+homepage: https://github.com/maltoze/logstash-output-clickhouse
 licenses:
 - Apache-2.0
 metadata: