logstash-output-clickhouse 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20cb59bb456f397ff9408209d676deb3c46214abbd874a5499ab5f7387bb80b9
-  data.tar.gz: 471cd0087a01acbcd6ed8efaa93e09f89102a2b2785bcfa7bb117e7ed8dd2b0d
+  metadata.gz: 34905df0e9293b2ff6471b904e018e8fe3d2d2f1bc372196422d5fa30107d50e
+  data.tar.gz: a20ec2301377b218b8395f6664be86f4b978e6ad72c42280fa2bfc0c889c0569
 SHA512:
-  metadata.gz: 3319460b03965943afecff0c3b910ebfb57611566f02b255b7fb3afd6b2a7f6781b89990cf667286972ed27bb503b90833a300f20450a0ad3e637c2144b3f37e
-  data.tar.gz: 609792b63cc1aa8d073f272e9539458738914c4ed1a795b22e23d6a86724b6e9c7e0277b533d0e02fa03d40168799197a614d3208e129b8ba534a8bb3ea80aee
+  metadata.gz: ccbd71994f6b705088616315a3340312381fce0ccae1dc4f97b793d417041d987ede5962c6f256111d6a78620bed275997153c9fa37bc86ca4770c4642469726
+  data.tar.gz: e0b00ed0a0b0f1e8a664698cd0c1258d48d67bf872afe983b26d9134ddead047852bfecebaa7d9fa7668400c94de76d0e1bd1475b074f11308d0b87a7f3a6284

data/README.md

@@ -1,6 +1,8 @@
 # Logstash Plugin
 [![Gem Version](https://badge.fury.io/rb/logstash-output-clickhouse.svg)](https://badge.fury.io/rb/logstash-output-clickhouse)
 
+This repo forked from [here](https://github.com/funcmike/logstash-output-clickhouse).
+
 This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch). 
 
 It has been modified to support ClickHouse JSON Format, but also supports fault tolerance.

data/lib/logstash/outputs/clickhouse.rb

@@ -58,7 +58,6 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
                  :idle_flush_time => @idle_flush_time,
                  :request_tokens => @pool_max,
                  :http_hosts => @http_hosts,
-                 :http_query => @http_query,
                  :headers => request_headers)
   end
 
@@ -74,8 +73,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     @pool_max.times { |t| @request_tokens << true }
     @requests = Array.new
 
-    params = { "query" => "INSERT INTO #{table} FORMAT JSONEachRow" }.merge(@extra_params)
-    @http_query = "?#{URI.encode_www_form(params)}"
+    @table_name = table
 
     buffer_initialize(
       :max_items => @flush_size,
@@ -91,6 +89,15 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
     buffer_receive(event)
   end
 
+  def generate_table_name(event)
+    event.sprintf(@table_name)
+  end
+
+  def generate_http_query(table_name)
+    params = { "query" => "INSERT INTO #{table_name} FORMAT JSONEachRow" }.merge(@extra_params)
+    return "?#{URI.encode_www_form(params)}"
+  end
+
   def mutate(src)
     return src if @mutations.empty?
     res = {}
@@ -115,15 +122,19 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
   public
 
   def flush(events, close = false)
-    documents = ""  #this is the string of hashes that we push to Fusion as documents
+    documents = {}
 
     events.each do |event|
-      documents << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
+      table_name = generate_table_name(event)
+      documents[table_name] ||= ""
+      documents[table_name] << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
     end
 
     hosts = @http_hosts.clone
 
-    make_request(documents, hosts, @http_query, 1, 1, hosts.sample)
+    documents.each do |table_name, document|
+      make_request(document, hosts, generate_http_query(table_name), 1, 1, hosts.sample)
+    end
   end
 
   private

data/logstash-output-clickhouse.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "logstash-output-clickhouse"
-  s.version = "0.1.4"
+  s.version = "0.1.5"
   s.licenses = ["Apache-2.0"]
   s.summary = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-clickhouse
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - kmajk
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-27 00:00:00.000000000 Z
+date: 2023-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api