logstash-output-clickhouse 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05f2b2d96799587dc5678cd80a6ef5d0c62a103797a298dfbe86acb1dbe09b89
-  data.tar.gz: ffeb98ddb2e27497ff54e9304180a2a44e88d093ce33755f842409a2101d4153
+  metadata.gz: 34905df0e9293b2ff6471b904e018e8fe3d2d2f1bc372196422d5fa30107d50e
+  data.tar.gz: a20ec2301377b218b8395f6664be86f4b978e6ad72c42280fa2bfc0c889c0569
 SHA512:
-  metadata.gz: 1c6d244d35441f46a0b49b79d1f278004e634334349d3949ec8d49e96fb02e31c942d13f8be72e7ddcd9996687acf81f1378b3587cbab6c6a9a9f93eb6634a6a
-  data.tar.gz: 530b73e05f582bf9b9af8a3be95aa1139fd0b21c3a75b411635453fa9c5e4bc538f8fb8477ab2388fad2560570c92a3623d614c97e69f103850ef6e54d45a23a
+  metadata.gz: ccbd71994f6b705088616315a3340312381fce0ccae1dc4f97b793d417041d987ede5962c6f256111d6a78620bed275997153c9fa37bc86ca4770c4642469726
+  data.tar.gz: e0b00ed0a0b0f1e8a664698cd0c1258d48d67bf872afe983b26d9134ddead047852bfecebaa7d9fa7668400c94de76d0e1bd1475b074f11308d0b87a7f3a6284

data/README.md

@@ -1,4 +1,7 @@
 # Logstash Plugin
+[![Gem Version](https://badge.fury.io/rb/logstash-output-clickhouse.svg)](https://badge.fury.io/rb/logstash-output-clickhouse)
+
+This repo forked from [here](https://github.com/funcmike/logstash-output-clickhouse).
 
 This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch). 
 

data/lib/logstash/outputs/clickhouse.rb

@@ -2,7 +2,6 @@
 require "logstash/outputs/base"
 require "logstash/namespace"
 require "logstash/json"
-require "logstash/util/shortname_resolver"
 require "uri"
 require "stud/buffer"
 require "logstash/plugin_mixins/http_client"
@@ -59,7 +58,6 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
                  :idle_flush_time => @idle_flush_time,
                  :request_tokens => @pool_max,
                  :http_hosts => @http_hosts,
-                 :http_query => @http_query,
                  :headers => request_headers)
   end
 
@@ -75,12 +73,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     @pool_max.times { |t| @request_tokens << true }
     @requests = Array.new
 
-    params = { "query" => "INSERT INTO #{table} FORMAT JSONEachRow" }.merge(@extra_params)
-    @http_query = "?#{URI.encode_www_form(params)}"
-
-    @hostnames_pool =
-      parse_http_hosts(http_hosts,
-                       ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
+    @table_name = table
 
     buffer_initialize(
       :max_items => @flush_size,
@@ -91,42 +84,18 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     print_plugin_info()
   end # def register
 
-  private
-
-  def parse_http_hosts(hosts, resolver)
-    ip_re = /^[\d]+\.[\d]+\.[\d]+\.[\d]+$/
-
-    lambda {
-      hosts.flat_map { |h|
-        scheme = URI(h).scheme
-        host = URI(h).host
-        port = URI(h).port
-        path = URI(h).path
-
-        if ip_re !~ host
-          resolver.get_addresses(host).map { |ip|
-            "#{scheme}://#{ip}:#{port}#{path}"
-          }
-        else
-          [h]
-        end
-      }
-    }
+  # This module currently does not support parallel requests as that would circumvent the batching
+  def receive(event)
+    buffer_receive(event)
   end
 
-  private
-
-  def get_host_addresses()
-    begin
-      @hostnames_pool.call
-    rescue Exception => ex
-      @logger.error("Error while resolving host", :error => ex.to_s)
-    end
+  def generate_table_name(event)
+    event.sprintf(@table_name)
   end
 
-  # This module currently does not support parallel requests as that would circumvent the batching
-  def receive(event)
-    buffer_receive(event)
+  def generate_http_query(table_name)
+    params = { "query" => "INSERT INTO #{table_name} FORMAT JSONEachRow" }.merge(@extra_params)
+    return "?#{URI.encode_www_form(params)}"
   end
 
   def mutate(src)
@@ -153,15 +122,19 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   public
 
   def flush(events, close = false)
-    documents = ""  #this is the string of hashes that we push to Fusion as documents
+    documents = {}
 
     events.each do |event|
-      documents << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
+      table_name = generate_table_name(event)
+      documents[table_name] ||= ""
+      documents[table_name] << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
     end
 
-    hosts = get_host_addresses()
+    hosts = @http_hosts.clone
 
-    make_request(documents, hosts, @http_query, 1, 1, hosts.sample)
+    documents.each do |table_name, document|
+      make_request(document, hosts, generate_http_query(table_name), 1, 1, hosts.sample)
+    end
   end
 
   private

data/logstash-output-clickhouse.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "logstash-output-clickhouse"
-  s.version = "0.1.3"
+  s.version = "0.1.5"
   s.licenses = ["Apache-2.0"]
   s.summary = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-clickhouse
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - kmajk
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-15 00:00:00.000000000 Z
+date: 2023-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -125,7 +125,6 @@ files:
 - LICENSE
 - README.md
 - lib/logstash/outputs/clickhouse.rb
-- lib/logstash/util/shortname_resolver.rb
 - logstash-output-clickhouse.gemspec
 homepage: https://github.com/maltoze/logstash-output-clickhouse
 licenses:

data/lib/logstash/util/shortname_resolver.rb

@@ -1,40 +0,0 @@
-require 'resolv'
-require 'mini_cache'
-
-class ShortNameResolver
-  def initialize(ttl:, logger:)
-    @ttl = ttl
-    @store = MiniCache::Store.new
-    @logger = logger
-  end
-
-  private
-  def resolve_cached(shortname)
-    @store.get_or_set(shortname) do
-      addresses = resolve(shortname)
-      raise "Bad shortname '#{shortname}'" if addresses.empty?
-      MiniCache::Data.new(addresses, expires_in: @ttl)
-    end
-  end
-
-  private
-  def resolve(shortname)
-    addresses = Resolv::DNS.open do |dns|
-      dns.getaddresses(shortname).map { |r| r.to_s }
-    end
-
-    @logger.info("Resolved shortname '#{shortname}' to addresses #{addresses}")
-
-    return addresses
-  end
-
-  public
-  def get_address(shortname)
-    return resolve_cached(shortname).sample
-  end
-
-  public
-  def get_addresses(shortname)
-    return resolve_cached(shortname)
-  end
-end