logstash-input-elastic_jdbc 0.2.1 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +53 -20
- data/lib/logstash/inputs/elastic_jdbc.rb +37 -6
- data/logstash-input-elastic-jdbc.gemspec +6 -4
- metadata +21 -8
- data/lib/logstash/inputs/data.json +0 -18
- data/lib/logstash/inputs/statement.sql +0 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 956e0978c2ea4e70c4b8410a90fc610113f3c46c12afdec85014ce6ffcd2d1ae
|
4
|
+
data.tar.gz: c191601527d553ba277c4383122ba053c5fb02a836f77416b65b117260a36cab
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d89379f8a8a87ffde484cd105b1fb9fe3760498375ee3b8bf90e230da185feaeea199a4b942b74e61049035ad8ad076898aecd9f9e2a95ab9a61a90107a2d42e
|
7
|
+
data.tar.gz: ba304b692417cd56fdc4aba032c4a58be96033b44bd2b522aadf531e6156b3b64c5d95f388bb3912b7079283fdf571690b14fcb6fed7571d812d9d5c35b2cc8a
|
data/README.md
CHANGED
@@ -1,33 +1,66 @@
|
|
1
1
|
# Logstash Plugin
|
2
|
-
[GitHub](https://github.com/ernesrocker/
|
2
|
+
[GitHub](https://github.com/ernesrocker/logstash-input-elastic_jdbc).
|
3
3
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
4
|
|
5
5
|
It is fully free and fully open source.
|
6
6
|
|
7
|
+
## How install
|
8
|
+
```sh
|
9
|
+
sudo /usr/share/logstash/bin/logstash-plugin install logstash-input-elastic_jdbc.gem
|
10
|
+
```
|
11
|
+
|
7
12
|
## Documentation
|
8
|
-
This plugin inherit of elasticsearch input plugin, and added a tracking_column
|
13
|
+
This plugin inherit of elasticsearch(**ES**) input plugin, and added a tracking_column
|
9
14
|
using in jdbc input plugin for make a query to obtain the updates values
|
10
15
|
Sample :
|
11
|
-
|
12
|
-
|
13
|
-
elastic_jdbc
|
16
|
+
```logstash
|
17
|
+
input{
|
18
|
+
elastic_jdbc{
|
14
19
|
hosts => "localhost"
|
20
|
+
index => "documents"
|
15
21
|
tracking_column => "last_update"
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
22
|
+
query => '{"query":{"range":{"created":{"gte":"2021-08-13T00:17:58+00:00"}}}}'
|
23
|
+
last_run_metadata_path => "/opt/logstash/last_run/elastic_jdbc_documents"
|
24
|
+
}
|
25
|
+
}
|
26
|
+
filter {
|
27
|
+
}
|
28
|
+
output{
|
29
|
+
stdout{}
|
30
|
+
}
|
31
|
+
```
|
32
|
+
In the sample before, we read from ES cluster, **documents** index, where documents hits have last_update field as
|
33
|
+
a **date** type field (recommend use [Ingest pipelines](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/ingest.html)),
|
34
|
+
then we look for all documents that have a field value **last_update** greater than the value stored in `/opt/logstash/last_run/elastic jdbc_documents" `.
|
35
|
+
|
36
|
+
####Required parameters:
|
37
|
+
* `hosts`: ES cluster url
|
38
|
+
* `index`: ES index
|
39
|
+
* `tracking_column`: Date field to tracking in ES index
|
40
|
+
* `last_run_metadata_path` : File path where stored the last value from last hist readed from ES index. By the default have the date `1960-01-01`
|
41
|
+
|
42
|
+
####Optional parameters:
|
43
|
+
* All [logstash-input-elasticsearch](https://rubygems.org/gems/logstash-input-elasticsearch) parameters can use in this plugins.
|
44
|
+
* `query`: By the default we use a bool query where we get a hits with `tracking column` greater that last value stored in `last_run_metadata_path`.
|
45
|
+
you can insert a query, but keep in mind that your query always be appended with the default query ( *if you don't need search by tracking column,
|
46
|
+
please use [logstash-input-elasticsearch](https://rubygems.org/gems/logstash-input-elasticsearch) plugin*).
|
47
|
+
Sample, for this query parameter ``query => '{"query":{"range":{"created":{"gte":"2021-08-13T00:17:58+00:00"}}}}'``,
|
48
|
+
the final query using this plugin would be:
|
49
|
+
|
50
|
+
```{
|
51
|
+
"query":{
|
52
|
+
"bool":{
|
53
|
+
"must":[
|
54
|
+
{"range": {"last_update":{"gt": "date_time_value_stored"}}},
|
55
|
+
{"range":{"abonado_date":{"gte": "2021-08-13T00:17:58+00:00"}}}
|
56
|
+
]
|
57
|
+
}
|
58
|
+
},
|
59
|
+
sort: [{"last_update"=>{:order=>"asc"}}]
|
60
|
+
}
|
61
|
+
```
|
62
|
+
**Note:** If you insert a ranking attribute within the query, we always overwrite it with the ranking values shown above.
|
63
|
+
|
31
64
|
### 1. Plugin Developement and Testing
|
32
65
|
|
33
66
|
#### Code
|
@@ -22,6 +22,9 @@ require "time"
|
|
22
22
|
class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
23
23
|
config_name "elastic_jdbc"
|
24
24
|
|
25
|
+
# Overwrite query default of elasticsearch plugin. We build a default query in this plugins.
|
26
|
+
config :query, :validate => :string, :default => '{}'
|
27
|
+
|
25
28
|
#region tracking configuration
|
26
29
|
# Path to file with last run time
|
27
30
|
config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
|
@@ -45,8 +48,8 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
45
48
|
if @tracking_column.nil?
|
46
49
|
raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
|
47
50
|
end
|
51
|
+
@value_tracker = ValueTracking.build_last_value_tracker(self)
|
48
52
|
super
|
49
|
-
set_value_tracker(ValueTracking.build_last_value_tracker(self))
|
50
53
|
build_query
|
51
54
|
end # def register
|
52
55
|
|
@@ -55,12 +58,24 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
55
58
|
end
|
56
59
|
|
57
60
|
def build_query
|
61
|
+
input_query = @base_query
|
62
|
+
# Remove sort tag from base query. We only sort by tracking column
|
63
|
+
input_query.delete("sort")
|
58
64
|
time_now = Time.now.utc
|
59
65
|
last_value = @value_tracker ? Time.parse(@value_tracker.value.to_s).iso8601 : Time.parse(time_now).iso8601
|
60
66
|
column = @tracking_column.to_s
|
61
|
-
|
62
|
-
|
63
|
-
|
67
|
+
query_default = {query: { bool: { must: [ {range: {column => {gt: last_value.to_s}}} ]}}}
|
68
|
+
if !input_query.nil? and !input_query.empty?
|
69
|
+
query_conditions = input_query["query"]
|
70
|
+
if query_conditions
|
71
|
+
must_statement = query_default[:query][:bool][:must]
|
72
|
+
final_must_cond = must_statement.append(query_conditions)
|
73
|
+
query_default[:query][:bool][:must] = final_must_cond
|
74
|
+
end
|
75
|
+
end
|
76
|
+
sort_condition = [{column => {order: "asc"}}]
|
77
|
+
query_default[:sort] = sort_condition
|
78
|
+
@base_query = LogStash::Json.load(query_default.to_json)
|
64
79
|
end
|
65
80
|
|
66
81
|
def run(output_queue)
|
@@ -68,8 +83,24 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
68
83
|
end # def run
|
69
84
|
|
70
85
|
def do_run_slice(output_queue, slice_id=nil)
|
71
|
-
|
72
|
-
|
86
|
+
slice_query = @base_query
|
87
|
+
slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
|
88
|
+
|
89
|
+
slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
|
90
|
+
logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
|
91
|
+
r = search_request(slice_options)
|
92
|
+
|
93
|
+
r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
|
94
|
+
logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
|
95
|
+
|
96
|
+
has_hits = r['hits']['hits'].any?
|
97
|
+
|
98
|
+
while has_hits && r['_scroll_id'] && !stop?
|
99
|
+
r = process_next_scroll(output_queue, r['_scroll_id'])
|
100
|
+
logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
|
101
|
+
has_hits = r['has_hits']
|
102
|
+
end
|
103
|
+
logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
|
73
104
|
end
|
74
105
|
|
75
106
|
def push_hit(hit, output_queue)
|
@@ -1,11 +1,12 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |s|
|
3
3
|
s.name = 'logstash-input-elastic_jdbc'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '1.0.0'
|
5
5
|
s.licenses = ['Apache-2.0']
|
6
|
-
s.summary = '
|
7
|
-
s.description = 'This plugin
|
8
|
-
|
6
|
+
s.summary = 'Reads querys from Elasticsearch cluster and write last run file.'
|
7
|
+
s.description = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname.
|
8
|
+
This gem is not a stand-alone program. Also, this plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.'
|
9
|
+
s.homepage = 'https://github.com/ernesrocker/logstash-input-elastic_jdbc'
|
9
10
|
s.authors = ['Ernesto Soler Calaña']
|
10
11
|
s.email = 'ernes920825@gmail.com'
|
11
12
|
s.require_paths = ['lib']
|
@@ -23,4 +24,5 @@ Gem::Specification.new do |s|
|
|
23
24
|
s.add_runtime_dependency 'logstash-codec-plain'
|
24
25
|
s.add_runtime_dependency 'stud', '>= 0.0.22'
|
25
26
|
s.add_development_dependency 'logstash-devutils', '~> 0.0', '>= 0.0.16'
|
27
|
+
s.add_development_dependency 'logstash-input-elasticsearch', '>= 4.3.1'
|
26
28
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elastic_jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ernesto Soler Calaña
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core-plugin-api
|
@@ -72,8 +72,23 @@ dependencies:
|
|
72
72
|
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 0.0.16
|
75
|
-
|
76
|
-
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: logstash-input-elasticsearch
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: 4.3.1
|
82
|
+
type: :development
|
83
|
+
prerelease: false
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 4.3.1
|
89
|
+
description: |-
|
90
|
+
This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname.
|
91
|
+
This gem is not a stand-alone program. Also, this plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.
|
77
92
|
email: ernes920825@gmail.com
|
78
93
|
executables: []
|
79
94
|
extensions: []
|
@@ -85,13 +100,11 @@ files:
|
|
85
100
|
- Gemfile
|
86
101
|
- LICENSE
|
87
102
|
- README.md
|
88
|
-
- lib/logstash/inputs/data.json
|
89
103
|
- lib/logstash/inputs/elastic_jdbc.rb
|
90
|
-
- lib/logstash/inputs/statement.sql
|
91
104
|
- lib/logstash/inputs/value_tracking.rb
|
92
105
|
- logstash-input-elastic-jdbc.gemspec
|
93
106
|
- spec/inputs/elastic-jdbc_spec.rb
|
94
|
-
homepage: https://github.com/ernesrocker/
|
107
|
+
homepage: https://github.com/ernesrocker/logstash-input-elastic_jdbc
|
95
108
|
licenses:
|
96
109
|
- Apache-2.0
|
97
110
|
metadata:
|
@@ -116,6 +129,6 @@ rubyforge_project:
|
|
116
129
|
rubygems_version: 2.7.6
|
117
130
|
signing_key:
|
118
131
|
specification_version: 4
|
119
|
-
summary:
|
132
|
+
summary: Reads querys from Elasticsearch cluster and write last run file.
|
120
133
|
test_files:
|
121
134
|
- spec/inputs/elastic-jdbc_spec.rb
|
@@ -1,18 +0,0 @@
|
|
1
|
-
[
|
2
|
-
{
|
3
|
-
"client_rut": "76591309-8",
|
4
|
-
"debtor_name": "LEIA SA",
|
5
|
-
"debtor_name_real": "LEIA SA",
|
6
|
-
"debtor_rut": "17316384-3",
|
7
|
-
"debtor_rut_real": "17316384-3",
|
8
|
-
"document_type": "33",
|
9
|
-
"expiration_days": 0,
|
10
|
-
"expiry_date": "2020-02-28",
|
11
|
-
"finance_amount": 0,
|
12
|
-
"finance_percent": 0,
|
13
|
-
"issue_date": "2019-09-25",
|
14
|
-
"issuer_rut": "76591309-8",
|
15
|
-
"number": "250",
|
16
|
-
"total_amount": 1606500
|
17
|
-
}
|
18
|
-
]
|
@@ -1 +0,0 @@
|
|
1
|
-
CREATE TABLE IF NOT EXISTS fc_documents (document_id varchar(256) COLLATE utf8_bin NOT NULL, channel json DEFAULT NULL, recipient_rut varchar(256) DEFAULT NULL, abonado_date datetime DEFAULT NULL, client_rut varchar(256) DEFAULT NULL, folio varchar(256) DEFAULT NULL, debtor_rut varchar(256) DEFAULT NULL, amount DOUBLE DEFAULT NULL, custom_expiration_utc datetime DEFAULT NULL, applied_base_rate double(20) DEFAULT NULL, applied_finance_percent double(20) DEFAULT NULL, emission_utc datetime DEFAULT NULL, emission_date datetime DEFAULT NULL, whenDate datetime DEFAULT NULL, expiration_date datetime DEFAULT NULL, backoffice_status varchar(256) DEFAULT NULL, action_id BIGINT DEFAULT null, admission_date datetime DEFAULT NULL, created BIGINT DEFAULT NULL, error varchar(256) DEFAULT NULL, issuer_rut varchar(256) DEFAULT NULL, operation_id varchar(256) DEFAULT NULL, paid_amount double(20) DEFAULT NULL, owner varchar(256) DEFAULT NULL, paid_date_utc datetime DEFAULT NULL, collect json DEFAULT NULL, comercial json DEFAULT NULL, debtor json DEFAULT NULL, sii_value json DEFAULT NULL, has_value json DEFAULT NULL, recipient json DEFAULT NULL, reception json DEFAULT NULL, referenceses json DEFAULT NULL, statuses json DEFAULT NULL, global_discounts_charges json DEFAULT NULL, inversor varchar(256) DEFAULT NULL, inversor_payment_date varchar(256) DEFAULT NULL, issuer json DEFAULT NULL, payment_classification json DEFAULT NULL, details json DEFAULT NULL, notification_contacts json DEFAULT NULL, verification_contacts json DEFAULT NULL, PRIMARY KEY (document_id));
|