logstash-output-clickhouse 0.1.3 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 05f2b2d96799587dc5678cd80a6ef5d0c62a103797a298dfbe86acb1dbe09b89
4
- data.tar.gz: ffeb98ddb2e27497ff54e9304180a2a44e88d093ce33755f842409a2101d4153
3
+ metadata.gz: 20cb59bb456f397ff9408209d676deb3c46214abbd874a5499ab5f7387bb80b9
4
+ data.tar.gz: 471cd0087a01acbcd6ed8efaa93e09f89102a2b2785bcfa7bb117e7ed8dd2b0d
5
5
  SHA512:
6
- metadata.gz: 1c6d244d35441f46a0b49b79d1f278004e634334349d3949ec8d49e96fb02e31c942d13f8be72e7ddcd9996687acf81f1378b3587cbab6c6a9a9f93eb6634a6a
7
- data.tar.gz: 530b73e05f582bf9b9af8a3be95aa1139fd0b21c3a75b411635453fa9c5e4bc538f8fb8477ab2388fad2560570c92a3623d614c97e69f103850ef6e54d45a23a
6
+ metadata.gz: 3319460b03965943afecff0c3b910ebfb57611566f02b255b7fb3afd6b2a7f6781b89990cf667286972ed27bb503b90833a300f20450a0ad3e637c2144b3f37e
7
+ data.tar.gz: 609792b63cc1aa8d073f272e9539458738914c4ed1a795b22e23d6a86724b6e9c7e0277b533d0e02fa03d40168799197a614d3208e129b8ba534a8bb3ea80aee
data/README.md CHANGED
@@ -1,4 +1,5 @@
1
1
  # Logstash Plugin
2
+ [![Gem Version](https://badge.fury.io/rb/logstash-output-clickhouse.svg)](https://badge.fury.io/rb/logstash-output-clickhouse)
2
3
 
3
4
  This plugin is a modified version of the Lucidworks logstash json_batch. That plugin is available [here](https://github.com/lucidworks/logstash-output-json_batch).
4
5
 
@@ -2,7 +2,6 @@
2
2
  require "logstash/outputs/base"
3
3
  require "logstash/namespace"
4
4
  require "logstash/json"
5
- require "logstash/util/shortname_resolver"
6
5
  require "uri"
7
6
  require "stud/buffer"
8
7
  require "logstash/plugin_mixins/http_client"
@@ -78,10 +77,6 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
78
77
  params = { "query" => "INSERT INTO #{table} FORMAT JSONEachRow" }.merge(@extra_params)
79
78
  @http_query = "?#{URI.encode_www_form(params)}"
80
79
 
81
- @hostnames_pool =
82
- parse_http_hosts(http_hosts,
83
- ShortNameResolver.new(ttl: @host_resolve_ttl_sec, logger: @logger))
84
-
85
80
  buffer_initialize(
86
81
  :max_items => @flush_size,
87
82
  :max_interval => @idle_flush_time,
@@ -91,39 +86,6 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
91
86
  print_plugin_info()
92
87
  end # def register
93
88
 
94
- private
95
-
96
- def parse_http_hosts(hosts, resolver)
97
- ip_re = /^[\d]+\.[\d]+\.[\d]+\.[\d]+$/
98
-
99
- lambda {
100
- hosts.flat_map { |h|
101
- scheme = URI(h).scheme
102
- host = URI(h).host
103
- port = URI(h).port
104
- path = URI(h).path
105
-
106
- if ip_re !~ host
107
- resolver.get_addresses(host).map { |ip|
108
- "#{scheme}://#{ip}:#{port}#{path}"
109
- }
110
- else
111
- [h]
112
- end
113
- }
114
- }
115
- end
116
-
117
- private
118
-
119
- def get_host_addresses()
120
- begin
121
- @hostnames_pool.call
122
- rescue Exception => ex
123
- @logger.error("Error while resolving host", :error => ex.to_s)
124
- end
125
- end
126
-
127
89
  # This module currently does not support parallel requests as that would circumvent the batching
128
90
  def receive(event)
129
91
  buffer_receive(event)
@@ -159,7 +121,7 @@ class LogStash::Outputs::ClickHouse < LogStash::Outputs::Base
159
121
  documents << LogStash::Json.dump(mutate(event.to_hash())) << "\n"
160
122
  end
161
123
 
162
- hosts = get_host_addresses()
124
+ hosts = @http_hosts.clone
163
125
 
164
126
  make_request(documents, hosts, @http_query, 1, 1, hosts.sample)
165
127
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = "logstash-output-clickhouse"
3
- s.version = "0.1.3"
3
+ s.version = "0.1.4"
4
4
  s.licenses = ["Apache-2.0"]
5
5
  s.summary = "This output lets you `POST` messages as JSON in a batched fashion to ClickHouse HTTP endpoint"
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-clickhouse
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - kmajk
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-07-15 00:00:00.000000000 Z
12
+ date: 2023-04-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: logstash-core-plugin-api
@@ -125,7 +125,6 @@ files:
125
125
  - LICENSE
126
126
  - README.md
127
127
  - lib/logstash/outputs/clickhouse.rb
128
- - lib/logstash/util/shortname_resolver.rb
129
128
  - logstash-output-clickhouse.gemspec
130
129
  homepage: https://github.com/maltoze/logstash-output-clickhouse
131
130
  licenses:
@@ -1,40 +0,0 @@
1
- require 'resolv'
2
- require 'mini_cache'
3
-
4
- class ShortNameResolver
5
- def initialize(ttl:, logger:)
6
- @ttl = ttl
7
- @store = MiniCache::Store.new
8
- @logger = logger
9
- end
10
-
11
- private
12
- def resolve_cached(shortname)
13
- @store.get_or_set(shortname) do
14
- addresses = resolve(shortname)
15
- raise "Bad shortname '#{shortname}'" if addresses.empty?
16
- MiniCache::Data.new(addresses, expires_in: @ttl)
17
- end
18
- end
19
-
20
- private
21
- def resolve(shortname)
22
- addresses = Resolv::DNS.open do |dns|
23
- dns.getaddresses(shortname).map { |r| r.to_s }
24
- end
25
-
26
- @logger.info("Resolved shortname '#{shortname}' to addresses #{addresses}")
27
-
28
- return addresses
29
- end
30
-
31
- public
32
- def get_address(shortname)
33
- return resolve_cached(shortname).sample
34
- end
35
-
36
- public
37
- def get_addresses(shortname)
38
- return resolve_cached(shortname)
39
- end
40
- end