logstash-input-elastic_jdbc 0.2.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 685b4a276223412feb5fa0fa6172fe33a6560c23a7efce5a393b11b692aac75c
-  data.tar.gz: 6cc1626519553b48b84bfb9524ed53449e4e7c15556d8d8244384362df8dfe69
+  metadata.gz: 956e0978c2ea4e70c4b8410a90fc610113f3c46c12afdec85014ce6ffcd2d1ae
+  data.tar.gz: c191601527d553ba277c4383122ba053c5fb02a836f77416b65b117260a36cab
 SHA512:
-  metadata.gz: fc5d3a185eea4f4aed1ff1cad0dac5596681b91175e8802c60fb7c1c72983db838622ac6c2a4563beee9031125129289dc2990ab61e5326f93a238b2e00d9116
-  data.tar.gz: 86e12c075705ae262a1ed6abd04cd84d180e505bf4ec247c64677fd110b04d36cc31e923515a31f0c596c2611cf81a4ffc13f85e7b28553de8fb1feb6c4c9662
+  metadata.gz: d89379f8a8a87ffde484cd105b1fb9fe3760498375ee3b8bf90e230da185feaeea199a4b942b74e61049035ad8ad076898aecd9f9e2a95ab9a61a90107a2d42e
+  data.tar.gz: ba304b692417cd56fdc4aba032c4a58be96033b44bd2b522aadf531e6156b3b64c5d95f388bb3912b7079283fdf571690b14fcb6fed7571d812d9d5c35b2cc8a

data/README.md

@@ -1,33 +1,66 @@
 # Logstash Plugin
-[GitHub](https://github.com/ernesrocker/RubyGems).
+[GitHub](https://github.com/ernesrocker/logstash-input-elastic_jdbc).
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source.
 
+## How install
+```sh
+sudo /usr/share/logstash/bin/logstash-plugin install logstash-input-elastic_jdbc.gem
+```
+
 ## Documentation
-This plugin inherit of elasticsearch input plugin, and added a tracking_column
+This plugin inherit of elasticsearch(**ES**) input plugin, and added a tracking_column
 using in jdbc input plugin for make a query to obtain the updates values
 Sample :
- input {
-       # Read all documents from Elasticsearch matching the given query
-       elastic_jdbc {
+```logstash
+  input{
+       elastic_jdbc{
          hosts => "localhost"
+         index => "documents"
          tracking_column => "last_update"
- 	            last_run_metadata_path => "/path_file"
- 	    }
-     }
-
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/).
-
-- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide
-
-## Need Help?
-
-Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
-
-## Developing
-
+         query => '{"query":{"range":{"created":{"gte":"2021-08-13T00:17:58+00:00"}}}}'
+         last_run_metadata_path => "/opt/logstash/last_run/elastic_jdbc_documents"
+       }
+  }
+  filter {
+  }
+  output{
+    stdout{}
+  }
+```  
+In the sample before, we read from ES cluster, **documents** index, where documents hits have last_update field as 
+a **date** type field (recommend use [Ingest pipelines](https://www.elastic.co/guide/en/elasticsearch/reference/7.x/ingest.html)),
+then we look for all documents that have a field value **last_update** greater than the value stored in `/opt/logstash/last_run/elastic jdbc_documents" `.
+
+####Required parameters:
+   * `hosts`: ES cluster url
+   * `index`: ES index
+   * `tracking_column`: Date field to tracking in ES index
+   * `last_run_metadata_path` : File path where stored the last value from last hist readed from ES index. By the default have the date `1960-01-01`
+
+####Optional parameters:
+   * All [logstash-input-elasticsearch](https://rubygems.org/gems/logstash-input-elasticsearch) parameters can use in this plugins.
+   * `query`: By the default we use a bool query where we get a hits with `tracking column` greater that last value stored in `last_run_metadata_path`. 
+   you can insert a query, but keep in mind that your query always be appended with the default query ( *if you don't need search by tracking column,
+   please use [logstash-input-elasticsearch](https://rubygems.org/gems/logstash-input-elasticsearch) plugin*). 
+   Sample, for this query parameter ``query => '{"query":{"range":{"created":{"gte":"2021-08-13T00:17:58+00:00"}}}}'``, 
+   the final query using this plugin would be:
+    
+   ```{
+       "query":{
+          "bool":{
+             "must":[
+                  {"range": {"last_update":{"gt": "date_time_value_stored"}}},
+                  {"range":{"abonado_date":{"gte": "2021-08-13T00:17:58+00:00"}}}
+             ]
+          }
+       }, 
+       sort: [{"last_update"=>{:order=>"asc"}}]
+      }
+   ```
+   **Note:** If you insert a ranking attribute within the query, we always overwrite it with the ranking values shown above.
+    
 ### 1. Plugin Developement and Testing
 
 #### Code

data/lib/logstash/inputs/elastic_jdbc.rb

@@ -22,6 +22,9 @@ require "time"
 class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
   config_name "elastic_jdbc"
 
+  # Overwrite query default of elasticsearch plugin. We build a default query in this plugins.
+  config :query, :validate => :string, :default => '{}'
+
   #region tracking configuration
   # Path to file with last run time
   config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
@@ -45,8 +48,8 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
     if @tracking_column.nil?
       raise(LogStash::ConfigurationError, "Must set :tracking_column if :use_column_value is true.")
     end
+    @value_tracker = ValueTracking.build_last_value_tracker(self)
     super
-    set_value_tracker(ValueTracking.build_last_value_tracker(self))
     build_query
   end # def register
 
@@ -55,12 +58,24 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
   end
 
   def build_query
+    input_query = @base_query
+    # Remove sort tag from base query. We only sort by tracking column
+    input_query.delete("sort")
     time_now = Time.now.utc
     last_value = @value_tracker ? Time.parse(@value_tracker.value.to_s).iso8601 : Time.parse(time_now).iso8601
     column = @tracking_column.to_s
-    query = {query: { range: {column => {gt: last_value.to_s}}}, sort: [{column => {order: "asc"}}]}
-    @query = query.to_json
-    @base_query = LogStash::Json.load(@query)
+    query_default = {query: { bool: { must: [ {range: {column => {gt: last_value.to_s}}} ]}}}
+    if !input_query.nil? and !input_query.empty?
+      query_conditions = input_query["query"] 
+      if query_conditions
+        must_statement = query_default[:query][:bool][:must]
+        final_must_cond = must_statement.append(query_conditions)
+        query_default[:query][:bool][:must] = final_must_cond
+      end
+    end
+    sort_condition = [{column => {order: "asc"}}]
+    query_default[:sort] = sort_condition
+    @base_query = LogStash::Json.load(query_default.to_json)
   end
 
   def run(output_queue)
@@ -68,8 +83,24 @@ class LogStash::Inputs::ElasticJdbc < LogStash::Inputs::Elasticsearch
   end # def run
 
   def do_run_slice(output_queue, slice_id=nil)
-    build_query
-    super
+    slice_query = @base_query
+    slice_query = slice_query.merge('slice' => { 'id' => slice_id, 'max' => @slices}) unless slice_id.nil?
+
+    slice_options = @options.merge(:body => LogStash::Json.dump(slice_query) )
+    logger.info("Slice starting", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+    r = search_request(slice_options)
+
+    r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }
+    logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+
+    has_hits = r['hits']['hits'].any?
+
+    while has_hits && r['_scroll_id'] && !stop?
+      r = process_next_scroll(output_queue, r['_scroll_id'])
+      logger.debug("Slice progress", slice_id: slice_id, slices: @slices) unless slice_id.nil?
+      has_hits = r['has_hits']
+    end
+    logger.info("Slice complete", slice_id: slice_id, slices: @slices) unless slice_id.nil?
   end
 
   def push_hit(hit, output_queue)

data/logstash-input-elastic-jdbc.gemspec

@@ -1,11 +1,12 @@
 # coding: utf-8
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-elastic_jdbc'
-  s.version       = '0.2.1'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
-  s.summary       = 'Logstash elastic_jdbc'
-  s.description   = 'This plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.'
-  s.homepage      = 'https://github.com/ernesrocker/RubyGems'
+  s.summary       = 'Reads querys from Elasticsearch cluster and write last run file.'
+  s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname.
+                     This gem is not a stand-alone program. Also, this plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.'
+  s.homepage      = 'https://github.com/ernesrocker/logstash-input-elastic_jdbc'
   s.authors       = ['Ernesto Soler Calaña']
   s.email         = 'ernes920825@gmail.com'
   s.require_paths = ['lib']
@@ -23,4 +24,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
   s.add_development_dependency 'logstash-devutils', '~> 0.0', '>= 0.0.16'
+  s.add_development_dependency 'logstash-input-elasticsearch', '>= 4.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elastic_jdbc
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 1.0.0
 platform: ruby
 authors:
 - Ernesto Soler Calaña
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-14 00:00:00.000000000 Z
+date: 2021-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -72,8 +72,23 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.16
-description: This plugin inherit of elasticsearch input plugin, but added tracking_column
-  like jdbc input plugin.
+- !ruby/object:Gem::Dependency
+  name: logstash-input-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.3.1
+description: |-
+  This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname.
+                       This gem is not a stand-alone program. Also, this plugin inherit of elasticsearch input plugin, but added tracking_column like jdbc input plugin.
 email: ernes920825@gmail.com
 executables: []
 extensions: []
@@ -85,13 +100,11 @@ files:
 - Gemfile
 - LICENSE
 - README.md
-- lib/logstash/inputs/data.json
 - lib/logstash/inputs/elastic_jdbc.rb
-- lib/logstash/inputs/statement.sql
 - lib/logstash/inputs/value_tracking.rb
 - logstash-input-elastic-jdbc.gemspec
 - spec/inputs/elastic-jdbc_spec.rb
-homepage: https://github.com/ernesrocker/RubyGems
+homepage: https://github.com/ernesrocker/logstash-input-elastic_jdbc
 licenses:
 - Apache-2.0
 metadata:
@@ -116,6 +129,6 @@ rubyforge_project:
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: Logstash elastic_jdbc
+summary: Reads querys from Elasticsearch cluster and write last run file.
 test_files:
 - spec/inputs/elastic-jdbc_spec.rb

data/lib/logstash/inputs/data.json

@@ -1,18 +0,0 @@
-[
-  {
-    "client_rut": "76591309-8",
-    "debtor_name": "LEIA SA",
-    "debtor_name_real": "LEIA SA",
-    "debtor_rut": "17316384-3",
-    "debtor_rut_real": "17316384-3",
-    "document_type": "33",
-    "expiration_days": 0,
-    "expiry_date": "2020-02-28",
-    "finance_amount": 0,
-    "finance_percent": 0,
-    "issue_date": "2019-09-25",
-    "issuer_rut": "76591309-8",
-    "number": "250",
-    "total_amount": 1606500
-  }
-]

data/lib/logstash/inputs/statement.sql

@@ -1 +0,0 @@
-CREATE TABLE IF NOT EXISTS fc_documents (document_id varchar(256) COLLATE utf8_bin NOT NULL, channel json DEFAULT NULL, recipient_rut varchar(256) DEFAULT NULL, abonado_date datetime DEFAULT NULL, client_rut varchar(256) DEFAULT NULL, folio varchar(256) DEFAULT NULL, debtor_rut varchar(256) DEFAULT NULL, amount DOUBLE DEFAULT NULL, custom_expiration_utc datetime DEFAULT NULL, applied_base_rate double(20) DEFAULT NULL, applied_finance_percent double(20) DEFAULT NULL, emission_utc datetime DEFAULT NULL, emission_date datetime DEFAULT NULL, whenDate datetime DEFAULT NULL, expiration_date datetime DEFAULT NULL, backoffice_status varchar(256) DEFAULT NULL, action_id BIGINT DEFAULT null, admission_date datetime DEFAULT NULL, created BIGINT DEFAULT NULL, error varchar(256) DEFAULT NULL, issuer_rut varchar(256) DEFAULT NULL, operation_id varchar(256) DEFAULT NULL, paid_amount double(20) DEFAULT NULL, owner varchar(256) DEFAULT NULL, paid_date_utc datetime DEFAULT NULL, collect json DEFAULT NULL, comercial json DEFAULT NULL, debtor json DEFAULT NULL, sii_value json DEFAULT NULL, has_value json DEFAULT NULL, recipient json DEFAULT NULL, reception json DEFAULT NULL, referenceses json DEFAULT NULL, statuses json DEFAULT NULL, global_discounts_charges json DEFAULT NULL, inversor varchar(256) DEFAULT NULL, inversor_payment_date varchar(256) DEFAULT NULL, issuer json DEFAULT NULL, payment_classification json DEFAULT NULL, details json DEFAULT NULL, notification_contacts json DEFAULT NULL, verification_contacts json DEFAULT NULL, PRIMARY KEY (document_id));