logstash-input-http_poller 2.0.6 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 10e023a2848c2a827b18e11fc0db42dd6d257d70
-  data.tar.gz: a014c52d6aa4509e877f064c214ed8e5dac2e803
+  metadata.gz: 791035a6079e584fae5b891675e5e17225405a70
+  data.tar.gz: 8286793c8788fc5af3c0d90ee10f613d49933e25
 SHA512:
-  metadata.gz: d85efa4810a99e15900b5387266c93ed81b58bdd11e0e89059bfb3aee2358524cc888ebcecef3c5bbf6151103004e46306482683d36d3ada0a68d618211ac1f0
-  data.tar.gz: 822a8f6881d29c803dc7328188c25cedbb456230312cbf61e520913f8db32c30cac7ef532b6fa494637fcd741b9b5e22861600f06ff5012502f234350c0cee81
+  metadata.gz: 95894f281b1440dd9c07bdf4790a2bf3017791c9c1f038a14b735637cb7a9b28db992c4e30898f29b37bbd158b51bc61ed159ad2d7fdbf5d2b68a45fac8303c3
+  data.tar.gz: 2b93713dad3146739aaed0a0b7ba838381d8e7b46ef9fa9a2a217e83d40b063d9ce4a04221c9d5697542aab77960572e8bbe7274776a1d416d30b8035b0b0be7

data/README.md

@@ -63,7 +63,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -81,7 +86,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/inputs/http_poller.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "logstash/plugin_mixins/http_client"
 require "socket" # for Socket.gethostname
 require "manticore"
+require "rufus/scheduler"
 
 # This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and
 # send them on their merry way. The idea behind this plugins came from a need to read springboot
@@ -33,7 +34,8 @@ require "manticore"
 #       }
 #     }
 #     request_timeout => 60
-#     interval => 60
+#     # Supports "cron", "every", "at" and "in" schedules by rufus scheduler
+#     schedule => { cron => "* * * * * UTC"}
 #     codec => "json"
 #     # A hash of request metadata info (timing, response headers, etc.) will be sent here
 #     metadata_target => "http_poller_metadata"
@@ -59,7 +61,20 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :urls, :validate => :hash, :required => true
 
   # How often (in seconds) the urls will be called
-  config :interval, :validate => :number, :required => true
+  # DEPRECATED. Use 'schedule' option instead.
+  # If both interval and schedule options are specified, interval
+  # option takes higher precedence
+  config :interval, :validate => :number, :deprecated => true
+
+  # Schedule of when to periodically poll from the urls
+  # Format: A hash with
+  #   + key: "cron" | "every" | "in" | "at"
+  #   + value: string
+  # Examples:
+  #   a) { "every" => "1h" }
+  #   b) { "cron" => "* * * * * UTC" }
+  # See: rufus/scheduler for details about different schedule options and value string format
+  config :schedule, :validate => :hash
 
   # Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
   config :target, :validate => :string
@@ -70,17 +85,19 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :metadata_target, :validate => :string, :default => '@metadata'
 
   public
+  Schedule_types = %w(cron every at in)
   def register
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
     @logger.info("Registering http_poller Input", :type => @type,
-                 :urls => @urls, :interval => @interval, :timeout => @timeout)
+                 :urls => @urls, :interval => @interval, :schedule => @schedule, :timeout => @timeout)
 
     setup_requests!
   end
 
   def stop
     Stud.stop!(@interval_thread) if @interval_thread
+    @scheduler.stop if @scheduler
   end
 
   private
@@ -133,13 +150,46 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   public
   def run(queue)
+    #interval or schedule must be provided. Must be exclusively either one. Not neither. Not both.
+    raise LogStash::ConfigurationError, "Invalid config. Neither interval nor schedule was specified." \
+      unless @interval ||  @schedule
+    raise LogStash::ConfigurationError, "Invalid config. Specify only interval or schedule. Not both." \
+      if @interval && @schedule
+
+    if @interval
+      setup_interval(queue)
+    elsif @schedule
+      setup_schedule(queue)
+    else
+      #should not reach here
+      raise LogStash::ConfigurationError, "Invalid config. Neither interval nor schedule was specified."
+    end
+  end
+
+  private
+  def setup_interval(queue)
     @interval_thread = Thread.current
     Stud.interval(@interval) do
       run_once(queue)
     end
   end
 
-  private
+  def setup_schedule(queue)
+    #schedule hash must contain exactly one of the allowed keys
+    msg_invalid_schedule = "Invalid config. schedule hash must contain " +
+      "exactly one of the following keys - cron, at, every or in"
+    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
+    schedule_type = @schedule.keys.first
+    schedule_value = @schedule[schedule_type]
+    raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
+
+    @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
+    #as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
+    opts = schedule_type == "every" ? { :first_in => 0.01 } : {} 
+    @scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
+    @scheduler.join
+  end
+
   def run_once(queue)
     @requests.each do |name, request|
       request_async(queue, name, request)

data/logstash-input-http_poller.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version     = '2.0.6'
+  s.version     = '2.1.0'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Poll HTTP endpoints with Logstash."
-  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors     = [ "Elastic", "andrewvc"]
   s.email       = 'info@elastic.co'
   s.homepage    = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -22,8 +22,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-mixin-http_client', ">= 2.2.4", "< 3.0.0"
   s.add_runtime_dependency 'stud', "~> 0.0.22"
+  s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
 
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
+  s.add_development_dependency 'timecop'
 end

data/spec/inputs/http_poller_spec.rb

@@ -1,11 +1,14 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/inputs/http_poller'
 require 'flores/random'
+require "timecop"
 
 describe LogStash::Inputs::HTTP_Poller do
   let(:metadata_target) { "_http_poller_metadata" }
   let(:queue) { Queue.new }
-  let(:default_interval) { 5 }
+  let(:default_schedule) {
+    { "cron" => "* * * * * UTC" }
+  }
   let(:default_name) { "url1 " }
   let(:default_url) { "http://localhost:1827" }
   let(:default_urls) {
@@ -15,7 +18,7 @@ describe LogStash::Inputs::HTTP_Poller do
   }
   let(:default_opts) {
     {
-      "interval" => default_interval,
+      "schedule" => default_schedule,
       "urls" => default_urls,
       "codec" => "json",
       "metadata_target" => metadata_target
@@ -31,9 +34,13 @@ describe LogStash::Inputs::HTTP_Poller do
     end
 
     describe "#run" do
-      it "should run at the specified interval" do
-        expect(Stud).to receive(:interval).with(default_interval).once
-        subject.run(double("queue"))
+      it "should setup a scheduler" do
+        runner = Thread.new do
+          subject.run(double("queue"))
+          expect(subject.instance_variable_get("@scheduler")).to be_a_kind_of(Rufus::Scheduler)
+        end
+        runner.kill
+        runner.join
       end
     end
 
@@ -151,6 +158,163 @@ describe LogStash::Inputs::HTTP_Poller do
     end
   end
 
+  describe "scheduler configuration" do
+    context "given an interval" do
+      let(:opts) {
+        {
+          "interval" => 2,
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should run once in each interval" do
+        instance = klass.new(opts)
+        instance.register
+        queue = Queue.new
+        runner = Thread.new do
+          instance.run(queue)
+        end
+        #T       0123456
+        #events  x x x x
+        #expects 3 events at T=5
+        sleep 5
+        instance.stop
+        runner.kill
+        runner.join
+        expect(queue.size).to eq(3)
+      end
+    end
+
+    context "given both interval and schedule options" do
+      let(:opts) {
+        {
+          "interval" => 1,
+          "schedule" => { "every" => "5s" },
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should raise ConfigurationError" do
+        instance = klass.new(opts)
+        instance.register
+        queue = Queue.new
+        runner = Thread.new do
+          expect{instance.run(queue)}.to raise_error(LogStash::ConfigurationError)
+        end
+        instance.stop
+        runner.kill
+        runner.join
+      end
+    end
+
+    context "given 'cron' expression" do
+      let(:opts) {
+        {
+          "schedule" => { "cron" => "* * * * * UTC" },
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should run at the schedule" do
+        instance = klass.new(opts)
+        instance.register
+        Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
+        Timecop.scale(60)
+        queue = Queue.new
+        runner = Thread.new do
+          instance.run(queue)
+        end
+        sleep 3
+        instance.stop
+        runner.kill
+        runner.join
+        expect(queue.size).to eq(2)
+        Timecop.return
+      end
+    end
+
+    context "given 'at' expression" do
+      let(:opts) {
+        {
+          "schedule" => { "at" => "2000-01-01 00:05:00 +0000"},
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should run at the schedule" do
+        instance = klass.new(opts)
+        instance.register
+        Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
+        Timecop.scale(60 * 5)
+        queue = Queue.new
+        runner = Thread.new do
+          instance.run(queue)
+        end
+        sleep 2
+        instance.stop
+        runner.kill
+        runner.join
+        expect(queue.size).to eq(1)
+        Timecop.return
+      end
+    end
+
+    context "given 'every' expression" do
+      let(:opts) {
+        {
+          "schedule" => { "every" => "2s"},
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should run at the schedule" do
+        instance = klass.new(opts)
+        instance.register
+        queue = Queue.new
+        runner = Thread.new do
+          instance.run(queue)
+        end
+        #T       0123456
+        #events  x x x x
+        #expects 3 events at T=5
+        sleep 5
+        instance.stop
+        runner.kill
+        runner.join
+        expect(queue.size).to eq(3)
+      end
+    end
+
+    context "given 'in' expression" do
+      let(:opts) {
+        {
+          "schedule" => { "in" => "2s"},
+          "urls" => default_urls,
+          "codec" => "json",
+          "metadata_target" => metadata_target
+        }
+      }
+      it "should run at the schedule" do
+        instance = klass.new(opts)
+        instance.register
+        queue = Queue.new
+        runner = Thread.new do
+          instance.run(queue)
+        end
+        sleep 3
+        instance.stop
+        runner.kill
+        runner.join
+        expect(queue.size).to eq(1)
+      end
+    end
+  end
+
   describe "events" do
     shared_examples("matching metadata") {
       let(:metadata) { event[metadata_target] }
@@ -292,7 +456,9 @@ describe LogStash::Inputs::HTTP_Poller do
         }
         let(:opts) {
           {
-            "interval" => default_interval,
+            "schedule" => {
+              "cron" => "* * * * * UTC"
+              },
             "urls" => {
               default_name => url
             },

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 2.0.6
+  version: 2.1.0
 platform: ruby
 authors:
 - Elastic
@@ -9,113 +9,141 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-11 00:00:00.000000000 Z
+date: 2016-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.2.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-mixin-http_client
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.4
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.2.4
-    - - <
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: 0.0.22
+  name: stud
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.22
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.22
+        version: 3.0.9
+  name: rufus-scheduler
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.9
 - !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   name: logstash-codec-json
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: flores
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: timecop
   prerelease: false
   type: :development
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program.
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -141,12 +169,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []