logstash-input-elastic_jdbc 0.2.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 685b4a276223412feb5fa0fa6172fe33a6560c23a7efce5a393b11b692aac75c
|
4
|
+
data.tar.gz: 6cc1626519553b48b84bfb9524ed53449e4e7c15556d8d8244384362df8dfe69
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc5d3a185eea4f4aed1ff1cad0dac5596681b91175e8802c60fb7c1c72983db838622ac6c2a4563beee9031125129289dc2990ab61e5326f93a238b2e00d9116
|
7
|
+
data.tar.gz: 86e12c075705ae262a1ed6abd04cd84d180e505bf4ec247c64677fd110b04d36cc31e923515a31f0c596c2611cf81a4ffc13f85e7b28553de8fb1feb6c4c9662
|
@@ -0,0 +1,18 @@
|
|
1
|
+
[
|
2
|
+
{
|
3
|
+
"client_rut": "76591309-8",
|
4
|
+
"debtor_name": "LEIA SA",
|
5
|
+
"debtor_name_real": "LEIA SA",
|
6
|
+
"debtor_rut": "17316384-3",
|
7
|
+
"debtor_rut_real": "17316384-3",
|
8
|
+
"document_type": "33",
|
9
|
+
"expiration_days": 0,
|
10
|
+
"expiry_date": "2020-02-28",
|
11
|
+
"finance_amount": 0,
|
12
|
+
"finance_percent": 0,
|
13
|
+
"issue_date": "2019-09-25",
|
14
|
+
"issuer_rut": "76591309-8",
|
15
|
+
"number": "250",
|
16
|
+
"total_amount": 1606500
|
17
|
+
}
|
18
|
+
]
|
@@ -45,9 +45,9 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
45
45
|
if @tracking_column.nil?
|
46
46
|
raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
|
47
47
|
end
|
48
|
+
super
|
48
49
|
set_value_tracker(ValueTracking.build_last_value_tracker(self))
|
49
50
|
build_query
|
50
|
-
super
|
51
51
|
end # def register
|
52
52
|
|
53
53
|
def set_value_tracker(instance)
|
@@ -58,7 +58,7 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
58
58
|
time_now = Time.now.utc
|
59
59
|
last_value = @value_tracker ? Time.parse(@value_tracker.value.to_s).iso8601 : Time.parse(time_now).iso8601
|
60
60
|
column = @tracking_column.to_s
|
61
|
-
query = {query: { range: {column => {gt: last_value.to_s}}}, sort: ["
|
61
|
+
query = {query: { range: {column => {gt: last_value.to_s}}}, sort: [{column => {order: "asc"}}]}
|
62
62
|
@query = query.to_json
|
63
63
|
@base_query = LogStash::Json.load(@query)
|
64
64
|
end
|
@@ -73,8 +73,30 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def push_hit(hit, output_queue)
|
76
|
-
|
77
|
-
|
76
|
+
event = LogStash::Event.new(hit['_source'])
|
77
|
+
|
78
|
+
if @docinfo
|
79
|
+
# do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
|
80
|
+
docinfo_target = event.get(@docinfo_target) || {}
|
81
|
+
|
82
|
+
unless docinfo_target.is_a?(Hash)
|
83
|
+
@logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event)
|
84
|
+
|
85
|
+
# TODO: (colin) I am not sure raising is a good strategy here?
|
86
|
+
raise Exception.new("Elasticsearch input: incompatible event")
|
87
|
+
end
|
88
|
+
|
89
|
+
@docinfo_fields.each do |field|
|
90
|
+
docinfo_target[field] = hit[field]
|
91
|
+
end
|
92
|
+
|
93
|
+
event.set(@docinfo_target, docinfo_target)
|
94
|
+
end
|
95
|
+
|
96
|
+
decorate(event)
|
97
|
+
output_queue << event
|
98
|
+
# Write in the file the last_update value register in the event.
|
99
|
+
@value_tracker.set_value(event.get(@tracking_column))
|
78
100
|
@value_tracker.write
|
79
101
|
end
|
80
102
|
|
@@ -0,0 +1 @@
|
|
1
|
+
CREATE TABLE IF NOT EXISTS fc_documents (document_id varchar(256) COLLATE utf8_bin NOT NULL, channel json DEFAULT NULL, recipient_rut varchar(256) DEFAULT NULL, abonado_date datetime DEFAULT NULL, client_rut varchar(256) DEFAULT NULL, folio varchar(256) DEFAULT NULL, debtor_rut varchar(256) DEFAULT NULL, amount DOUBLE DEFAULT NULL, custom_expiration_utc datetime DEFAULT NULL, applied_base_rate double(20) DEFAULT NULL, applied_finance_percent double(20) DEFAULT NULL, emission_utc datetime DEFAULT NULL, emission_date datetime DEFAULT NULL, whenDate datetime DEFAULT NULL, expiration_date datetime DEFAULT NULL, backoffice_status varchar(256) DEFAULT NULL, action_id BIGINT DEFAULT null, admission_date datetime DEFAULT NULL, created BIGINT DEFAULT NULL, error varchar(256) DEFAULT NULL, issuer_rut varchar(256) DEFAULT NULL, operation_id varchar(256) DEFAULT NULL, paid_amount double(20) DEFAULT NULL, owner varchar(256) DEFAULT NULL, paid_date_utc datetime DEFAULT NULL, collect json DEFAULT NULL, comercial json DEFAULT NULL, debtor json DEFAULT NULL, sii_value json DEFAULT NULL, has_value json DEFAULT NULL, recipient json DEFAULT NULL, reception json DEFAULT NULL, referenceses json DEFAULT NULL, statuses json DEFAULT NULL, global_discounts_charges json DEFAULT NULL, inversor varchar(256) DEFAULT NULL, inversor_payment_date varchar(256) DEFAULT NULL, issuer json DEFAULT NULL, payment_classification json DEFAULT NULL, details json DEFAULT NULL, notification_contacts json DEFAULT NULL, verification_contacts json DEFAULT NULL, PRIMARY KEY (document_id));
|
@@ -1,6 +1,7 @@
|
|
1
|
+
# coding: utf-8
|
1
2
|
Gem::Specification.new do |s|
|
2
3
|
s.name = 'logstash-input-elastic_jdbc'
|
3
|
-
s.version = '0.2.
|
4
|
+
s.version = '0.2.1'
|
4
5
|
s.licenses = ['Apache-2.0']
|
5
6
|
s.summary = 'Logstash elastic_jdbc'
|
6
7
|
s.description = 'This plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-input-elastic_jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ernesto Soler Calaña
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-11-
|
11
|
+
date: 2019-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core-plugin-api
|
@@ -85,7 +85,9 @@ files:
|
|
85
85
|
- Gemfile
|
86
86
|
- LICENSE
|
87
87
|
- README.md
|
88
|
+
- lib/logstash/inputs/data.json
|
88
89
|
- lib/logstash/inputs/elastic_jdbc.rb
|
90
|
+
- lib/logstash/inputs/statement.sql
|
89
91
|
- lib/logstash/inputs/value_tracking.rb
|
90
92
|
- logstash-input-elastic-jdbc.gemspec
|
91
93
|
- spec/inputs/elastic-jdbc_spec.rb
|