logstash-input-elastic_jdbc 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11a398ad2179a95c86fcc2c0bae1bf57a6cc3004a98e883df5fc3e8f475a619b
-  data.tar.gz: d7b2a56f95b0e06d103c5b9040e1e59b47d5ab0190d7051ae08e0c70621118da
+  metadata.gz: 685b4a276223412feb5fa0fa6172fe33a6560c23a7efce5a393b11b692aac75c
+  data.tar.gz: 6cc1626519553b48b84bfb9524ed53449e4e7c15556d8d8244384362df8dfe69
 SHA512:
-  metadata.gz: f0fa08a3f7b4578901a6f46c1eafecaf72fe3f3d0ddd81a7c613dcd30192e759ce018e7794e18cd506af80772af87cc410b050b4d4af3e775841b65c5f0c9708
-  data.tar.gz: b82b80d7ab1f383af1bcf03f358a1ca7104b6a439ce60243e822213984931636df9c67e4f5aad0a64d61f19f6af01c9d2de3756185b06a15ef635a51dc2fcfd4
+  metadata.gz: fc5d3a185eea4f4aed1ff1cad0dac5596681b91175e8802c60fb7c1c72983db838622ac6c2a4563beee9031125129289dc2990ab61e5326f93a238b2e00d9116
+  data.tar.gz: 86e12c075705ae262a1ed6abd04cd84d180e505bf4ec247c64677fd110b04d36cc31e923515a31f0c596c2611cf81a4ffc13f85e7b28553de8fb1feb6c4c9662

data/lib/logstash/inputs/data.json

@@ -0,0 +1,18 @@
+[
+  {
+    "client_rut": "76591309-8",
+    "debtor_name": "LEIA SA",
+    "debtor_name_real": "LEIA SA",
+    "debtor_rut": "17316384-3",
+    "debtor_rut_real": "17316384-3",
+    "document_type": "33",
+    "expiration_days": 0,
+    "expiry_date": "2020-02-28",
+    "finance_amount": 0,
+    "finance_percent": 0,
+    "issue_date": "2019-09-25",
+    "issuer_rut": "76591309-8",
+    "number": "250",
+    "total_amount": 1606500
+  }
+]

data/lib/logstash/inputs/elastic_jdbc.rb

@@ -45,9 +45,9 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
     if @tracking_column.nil?
       raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
     end
+    super
     set_value_tracker(ValueTracking.build_last_value_tracker(self))
     build_query
-    super
   end # def register
 
   def set_value_tracker(instance)
@@ -58,7 +58,7 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
     time_now = Time.now.utc
     last_value = @value_tracker ? Time.parse(@value_tracker.value.to_s).iso8601 : Time.parse(time_now).iso8601
     column = @tracking_column.to_s
-    query = {query: { range: {column => {gt: last_value.to_s}}}, sort: ["_doc"]}
+    query = {query: { range: {column => {gt: last_value.to_s}}}, sort: [{column => {order: "asc"}}]}
     @query = query.to_json
     @base_query = LogStash::Json.load(@query)
   end
@@ -73,8 +73,30 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
   end
 
   def push_hit(hit, output_queue)
-    super
-    @value_tracker.set_value(Time.now.to_s)
+    event = LogStash::Event.new(hit['_source'])
+
+    if @docinfo
+      # do not assume event[@docinfo_target] to be in-place updatable. first get it, update it, then at the end set it in the event.
+      docinfo_target = event.get(@docinfo_target) || {}
+
+      unless docinfo_target.is_a?(Hash)
+        @logger.error("Elasticsearch Input: Incompatible Event, incompatible type for the docinfo_target=#{@docinfo_target} field in the `_source` document, expected a hash got:", :docinfo_target_type => docinfo_target.class, :event => event)
+
+        # TODO: (colin) I am not sure raising is a good strategy here?
+        raise Exception.new("Elasticsearch input: incompatible event")
+      end
+
+      @docinfo_fields.each do |field|
+        docinfo_target[field] = hit[field]
+      end
+
+      event.set(@docinfo_target, docinfo_target)
+    end
+
+    decorate(event)
+    output_queue << event
+    # Write in the file the last_update value register in the event.
+    @value_tracker.set_value(event.get(@tracking_column))
     @value_tracker.write
   end
 

data/lib/logstash/inputs/statement.sql

@@ -0,0 +1 @@
+CREATE TABLE IF NOT EXISTS fc_documents (document_id varchar(256) COLLATE utf8_bin NOT NULL, channel json DEFAULT NULL, recipient_rut varchar(256) DEFAULT NULL, abonado_date datetime DEFAULT NULL, client_rut varchar(256) DEFAULT NULL, folio varchar(256) DEFAULT NULL, debtor_rut varchar(256) DEFAULT NULL, amount DOUBLE DEFAULT NULL, custom_expiration_utc datetime DEFAULT NULL, applied_base_rate double(20) DEFAULT NULL, applied_finance_percent double(20) DEFAULT NULL, emission_utc datetime DEFAULT NULL, emission_date datetime DEFAULT NULL, whenDate datetime DEFAULT NULL, expiration_date datetime DEFAULT NULL, backoffice_status varchar(256) DEFAULT NULL, action_id BIGINT DEFAULT null, admission_date datetime DEFAULT NULL, created BIGINT DEFAULT NULL, error varchar(256) DEFAULT NULL, issuer_rut varchar(256) DEFAULT NULL, operation_id varchar(256) DEFAULT NULL, paid_amount double(20) DEFAULT NULL, owner varchar(256) DEFAULT NULL, paid_date_utc datetime DEFAULT NULL, collect json DEFAULT NULL, comercial json DEFAULT NULL, debtor json DEFAULT NULL, sii_value json DEFAULT NULL, has_value json DEFAULT NULL, recipient json DEFAULT NULL, reception json DEFAULT NULL, referenceses json DEFAULT NULL, statuses json DEFAULT NULL, global_discounts_charges json DEFAULT NULL, inversor varchar(256) DEFAULT NULL, inversor_payment_date varchar(256) DEFAULT NULL, issuer json DEFAULT NULL, payment_classification json DEFAULT NULL, details json DEFAULT NULL, notification_contacts json DEFAULT NULL, verification_contacts json DEFAULT NULL, PRIMARY KEY (document_id));

data/logstash-input-elastic-jdbc.gemspec

@@ -1,6 +1,7 @@
+# coding: utf-8
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-elastic_jdbc'
-  s.version       = '0.2.0'
+  s.version       = '0.2.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Logstash elastic_jdbc'
   s.description   = 'This plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elastic_jdbc
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Ernesto Soler Calaña
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-12 00:00:00.000000000 Z
+date: 2019-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -85,7 +85,9 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/logstash/inputs/data.json
 - lib/logstash/inputs/elastic_jdbc.rb
+- lib/logstash/inputs/statement.sql
 - lib/logstash/inputs/value_tracking.rb
 - logstash-input-elastic-jdbc.gemspec
 - spec/inputs/elastic-jdbc_spec.rb