logstash-input-elasticsearch 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5af8082f8aad876b3e67645560651a1e3d4f7712f52619f138fe8fe8b312b8f
-  data.tar.gz: 0c0fbd86007d4ad7ed62366e5b8e326103539a1b0de8a65f6ce90f8c7796dfd6
+  metadata.gz: c800b8e0032e2d43897b1c4e0dab3cf4d6d22c14919fcdefe7b7df8ec4a64ffa
+  data.tar.gz: 063e672f41db0c0711c7d2229fd05b5dbef0d28fed9f34afe5c2c2ad57ff1dc2
 SHA512:
-  metadata.gz: 8010e3abb6ea0c5bf2bf3c0b25d66a541bdd9c99a23e5d654cadec5571291cfe94a076f628526ca407f8715b4c566d953f37d9194d1924e565f386249fb74e4c
-  data.tar.gz: 7f30e0c6da1bad57e60d6f072a4bea5d60226f98af399b4f45e984635c81c9717c58fb1212cecc6a9c846fc18593105180183b6f33539b20143ac7a8cccc1d4e
+  metadata.gz: 63503ac4073666f88eca84c6af52d2598ba425b4ca7b16ec6c77bcb97def7083119943ea7f940536bd49a8cad581d2afb502634a00e336a223f2abff9842038d
+  data.tar.gz: 5d2ec1e87c76a79f590d3f93dc8e2667597b74aba4f1503282f75de34ce98e6c6cac4d49ddcc86668669eb7706dc97a312fb93f7bad3b88fa2ff98608cd9bc94

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.2.0
+  - Docs: Deprecate `document_type`
+  - Add support for scheduling periodic execution of the query #81
+
 ## 4.1.1
   - Update gemspec summary
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -32,6 +32,9 @@ plugin to version 4.0.2 or higher.
 
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
+You can periodically schedule ingestion using a cron syntax 
+(see `schedule` setting) or run the query one time to load
+data into Logstash.
 
 Example:
 [source,ruby]
@@ -55,6 +58,25 @@ This would create an Elasticsearch query with the following format:
     }'
 
 
+==== Scheduling
+
+Input from this plugin can be scheduled to run periodically according to a specific
+schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
+The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
+
+Examples:
+
+|==========================================================
+| `* 5 * 1-3 *`               | will execute every minute of 5am every day of January through March.
+| `0 * * * *`                 | will execute on the 0th minute of every hour every day.
+| `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
+|==========================================================
+
+
+Further documentation describing this syntax can be found
+https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
+
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input Configuration Options
 
@@ -71,6 +93,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-query>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-schedule>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-scroll>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
@@ -127,6 +150,11 @@ Example
     }
 
 
+NOTE: Starting with Logstash 6.0, the `document_type` option is
+deprecated due to the
+https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
+It will be removed in the next major version of Logstash.
+
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
 ===== `docinfo_fields` 
 
@@ -136,7 +164,7 @@ Example
 If document metadata storage is requested by enabling the `docinfo`
 option, this option lists the metadata fields to save in the current
 event. See
-[Document Metadata](http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html)
+http://www.elasticsearch.org/guide/en/elasticsearch/guide/current/_document_metadata.html[Document Metadata]
 in the Elasticsearch documentation for more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
@@ -184,9 +212,21 @@ string authentication will be disabled.
   * Default value is `'{ "sort": [ "_doc" ] }'`
 
 The query to be executed. Read the
-[Elasticsearch query DSL documentation](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html)
+https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl.html[Elasticsearch query DSL documentation]
 for more information.
 
+[id="plugins-{type}s-{plugin}-schedule"]
+===== `schedule` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Schedule of when to periodically run statement, in Cron format
+for example: "* * * * *" (execute query every minute, on the minute)
+
+There is no schedule by default. If no schedule is given, then the statement is run
+exactly once.
+
 [id="plugins-{type}s-{plugin}-scroll"]
 ===== `scroll` 
 

data/lib/logstash/inputs/elasticsearch.rb

@@ -15,6 +15,8 @@ require "base64"
 # 
 # Read from an Elasticsearch cluster, based on search query results.
 # This is useful for replaying test logs, reindexing, etc.
+# It also supports periodically scheduling lookup enrichments
+# using a cron syntax (see `schedule` setting).
 #
 # Example:
 # [source,ruby]
@@ -37,6 +39,24 @@ require "base64"
 #       "sort": [ "_doc" ]
 #     }'
 #
+# ==== Scheduling
+#
+# Input from this plugin can be scheduled to run periodically according to a specific
+# schedule. This scheduling syntax is powered by https://github.com/jmettraux/rufus-scheduler[rufus-scheduler].
+# The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
+#
+# Examples:
+#
+# |==========================================================
+# | `* 5 * 1-3 *`               | will execute every minute of 5am every day of January through March.
+# | `0 * * * *`                 | will execute on the 0th minute of every hour every day.
+# | `0 6 * * * America/Chicago` | will execute at 6:00am (UTC/GMT -5) every day.
+# |==========================================================
+#
+#
+# Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here].
+#
+#
 class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   config_name "elasticsearch"
 
@@ -114,8 +134,16 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
   # SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary 
   config :ca_file, :validate => :path
 
+  # Schedule of when to periodically run statement, in Cron format
+  # for example: "* * * * *" (execute query every minute, on the minute)
+  #
+  # There is no schedule by default. If no schedule is given, then the statement is run
+  # exactly once.
+  config :schedule, :validate => :string
+
   def register
     require "elasticsearch"
+    require "rufus/scheduler"
 
     @options = {
       :index => @index,
@@ -147,7 +175,27 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     @client = Elasticsearch::Client.new(:hosts => hosts, :transport_options => transport_options)
   end
 
+
   def run(output_queue)
+    if @schedule
+      @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
+      @scheduler.cron @schedule do
+        do_run(output_queue)
+      end
+
+      @scheduler.join
+    else
+      do_run(output_queue)
+    end
+  end
+
+  def stop
+    @scheduler.stop if @scheduler
+  end
+
+  private
+
+  def do_run(output_queue)
     # get first wave of data
     r = @client.search(@options)
 
@@ -160,8 +208,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base
     end
   end
 
-  private
-
   def process_next_scroll(output_queue, scroll_id)
     r = scroll_request(scroll_id)
     r['hits']['hits'].each { |hit| push_hit(hit, output_queue) }

data/logstash-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-elasticsearch'
-  s.version         = '4.1.1'
+  s.version         = '4.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads query results from an Elasticsearch cluster"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,6 +25,12 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'elasticsearch', ['>= 5.0.3', '< 6.0.0']
 
   s.add_runtime_dependency 'logstash-codec-json'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'tzinfo'
+  s.add_runtime_dependency 'tzinfo-data'
+  s.add_runtime_dependency 'rufus-scheduler'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'timecop'
 end

data/spec/inputs/elasticsearch_spec.rb

@@ -2,12 +2,23 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/elasticsearch"
 require "elasticsearch"
+require "timecop"
+require "stud/temporary"
+require "time"
+require "date"
 
 describe LogStash::Inputs::Elasticsearch do
 
+  let(:plugin) { LogStash::Inputs::Elasticsearch.new(config) }
+  let(:queue) { Queue.new }
+
   it_behaves_like "an interruptible input plugin" do
     let(:esclient) { double("elasticsearch-client") }
-    let(:config) { { } }
+    let(:config) do
+      {
+        "schedule" => "* * * * * UTC"
+      }
+    end
 
     before :each do
       allow(Elasticsearch::Client).to receive(:new).and_return(esclient)
@@ -271,4 +282,66 @@ describe LogStash::Inputs::Elasticsearch do
       end
     end
   end
+
+  context "when scheduling" do
+    let(:config) do
+      {
+        "hosts" => ["localhost"],
+        "query" => '{ "query": { "match": { "city_name": "Okinawa" } }, "fields": ["message"] }',
+        "schedule" => "* * * * * UTC"
+      }
+    end
+
+    response = {
+      "_scroll_id" => "cXVlcnlUaGVuRmV0Y2g",
+      "took" => 27,
+      "timed_out" => false,
+      "_shards" => {
+        "total" => 169,
+        "successful" => 169,
+        "failed" => 0
+      },
+      "hits" => {
+        "total" => 1,
+        "max_score" => 1.0,
+        "hits" => [ {
+          "_index" => "logstash-2014.10.12",
+          "_type" => "logs",
+          "_id" => "C5b2xLQwTZa76jBmHIbwHQ",
+          "_score" => 1.0,
+          "_source" => { "message" => ["ohayo"] }
+        } ]
+      }
+    }
+
+    scroll_reponse = {
+      "_scroll_id" => "r453Wc1jh0caLJhSDg",
+      "hits" => { "hits" => [] }
+    }
+
+    before do
+      plugin.register
+    end
+
+    it "should properly schedule" do
+
+      Timecop.travel(Time.new(2000))
+      Timecop.scale(60)
+      runner = Thread.new do
+        expect(plugin).to receive(:do_run) {
+          queue << LogStash::Event.new({})
+        }.at_least(:twice)
+
+        plugin.run(queue)
+      end
+      sleep 3
+      plugin.stop
+      runner.kill
+      runner.join
+      expect(queue.size).to eq(2)
+      Timecop.return
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,6 +64,76 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: sequel
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tzinfo-data
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rufus-scheduler
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -78,6 +148,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -118,7 +202,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Reads query results from an Elasticsearch cluster