logstash-input-http_poller 2.1.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 791035a6079e584fae5b891675e5e17225405a70
-  data.tar.gz: 8286793c8788fc5af3c0d90ee10f613d49933e25
+  metadata.gz: 7c10834cd3780892bc9232ffb268b691d9f19d0e
+  data.tar.gz: d01112b2050202711def2a766c2d993bb4ebab66
 SHA512:
-  metadata.gz: 95894f281b1440dd9c07bdf4790a2bf3017791c9c1f038a14b735637cb7a9b28db992c4e30898f29b37bbd158b51bc61ed159ad2d7fdbf5d2b68a45fac8303c3
-  data.tar.gz: 2b93713dad3146739aaed0a0b7ba838381d8e7b46ef9fa9a2a217e83d40b063d9ce4a04221c9d5697542aab77960572e8bbe7274776a1d416d30b8035b0b0be7
+  metadata.gz: 3e19ccee8c2c9ef27489ce79fa020784d0b6f348aea0ff90921f80050ab709dcec8e96f94792c5b3b39ea7c69e844430969598043060ef3ae34c383ac1145d31
+  data.tar.gz: c292ca93af6d0ded87971e2f3cb529b48e1fbd401c9a6abdab762c588f5e1bbc5e8a959d38d0c8737066d1714470821c504e5a66d446d1c75cd02dc19adbea19

data/CHANGELOG.md

@@ -1,3 +1,5 @@
+## 3.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
 # 2.0.5
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.4

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec

data/lib/logstash/inputs/http_poller.rb

@@ -4,7 +4,6 @@ require "logstash/namespace"
 require "logstash/plugin_mixins/http_client"
 require "socket" # for Socket.gethostname
 require "manticore"
-require "rufus/scheduler"
 
 # This Logstash input plugin allows you to call an HTTP API, decode the output of it into event(s), and
 # send them on their merry way. The idea behind this plugins came from a need to read springboot
@@ -34,8 +33,7 @@ require "rufus/scheduler"
 #       }
 #     }
 #     request_timeout => 60
-#     # Supports "cron", "every", "at" and "in" schedules by rufus scheduler
-#     schedule => { cron => "* * * * * UTC"}
+#     interval => 60
 #     codec => "json"
 #     # A hash of request metadata info (timing, response headers, etc.) will be sent here
 #     metadata_target => "http_poller_metadata"
@@ -61,20 +59,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :urls, :validate => :hash, :required => true
 
   # How often (in seconds) the urls will be called
-  # DEPRECATED. Use 'schedule' option instead.
-  # If both interval and schedule options are specified, interval
-  # option takes higher precedence
-  config :interval, :validate => :number, :deprecated => true
-
-  # Schedule of when to periodically poll from the urls
-  # Format: A hash with
-  #   + key: "cron" | "every" | "in" | "at"
-  #   + value: string
-  # Examples:
-  #   a) { "every" => "1h" }
-  #   b) { "cron" => "* * * * * UTC" }
-  # See: rufus/scheduler for details about different schedule options and value string format
-  config :schedule, :validate => :hash
+  config :interval, :validate => :number, :required => true
 
   # Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
   config :target, :validate => :string
@@ -85,19 +70,17 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   config :metadata_target, :validate => :string, :default => '@metadata'
 
   public
-  Schedule_types = %w(cron every at in)
   def register
     @host = Socket.gethostname.force_encoding(Encoding::UTF_8)
 
     @logger.info("Registering http_poller Input", :type => @type,
-                 :urls => @urls, :interval => @interval, :schedule => @schedule, :timeout => @timeout)
+                 :urls => @urls, :interval => @interval, :timeout => @timeout)
 
     setup_requests!
   end
 
   def stop
     Stud.stop!(@interval_thread) if @interval_thread
-    @scheduler.stop if @scheduler
   end
 
   private
@@ -150,46 +133,13 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
   public
   def run(queue)
-    #interval or schedule must be provided. Must be exclusively either one. Not neither. Not both.
-    raise LogStash::ConfigurationError, "Invalid config. Neither interval nor schedule was specified." \
-      unless @interval ||  @schedule
-    raise LogStash::ConfigurationError, "Invalid config. Specify only interval or schedule. Not both." \
-      if @interval && @schedule
-
-    if @interval
-      setup_interval(queue)
-    elsif @schedule
-      setup_schedule(queue)
-    else
-      #should not reach here
-      raise LogStash::ConfigurationError, "Invalid config. Neither interval nor schedule was specified."
-    end
-  end
-
-  private
-  def setup_interval(queue)
     @interval_thread = Thread.current
     Stud.interval(@interval) do
       run_once(queue)
     end
   end
 
-  def setup_schedule(queue)
-    #schedule hash must contain exactly one of the allowed keys
-    msg_invalid_schedule = "Invalid config. schedule hash must contain " +
-      "exactly one of the following keys - cron, at, every or in"
-    raise Logstash::ConfigurationError, msg_invalid_schedule if @schedule.keys.length !=1
-    schedule_type = @schedule.keys.first
-    schedule_value = @schedule[schedule_type]
-    raise LogStash::ConfigurationError, msg_invalid_schedule unless Schedule_types.include?(schedule_type)
-
-    @scheduler = Rufus::Scheduler.new(:max_work_threads => 1)
-    #as of v3.0.9, :first_in => :now doesn't work. Use the following workaround instead
-    opts = schedule_type == "every" ? { :first_in => 0.01 } : {} 
-    @scheduler.send(schedule_type, schedule_value, opts) { run_once(queue) }
-    @scheduler.join
-  end
-
+  private
   def run_once(queue)
     @requests.each do |name, request|
       request_async(queue, name, request)
@@ -244,13 +194,13 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
 
     # This is also in the metadata, but we send it anyone because we want this
     # persisted by default, whereas metadata isn't. People don't like mysterious errors
-    event["http_request_failure"] = {
+    event.set("http_request_failure", {
       "request" => structure_request(request),
       "name" => name,
       "error" => exception.to_s,
       "backtrace" => exception.backtrace,
       "runtime_seconds" => execution_time
-   }
+   })
 
     queue << event
   rescue StandardError, java.lang.Exception => e
@@ -266,7 +216,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   private
   def apply_metadata(event, name, request, response=nil, execution_time=nil)
     return unless @metadata_target
-    event[@metadata_target] = event_metadata(name, request, response, execution_time)
+    event.set(@metadata_target, event_metadata(name, request, response, execution_time))
   end
 
   private

data/logstash-input-http_poller.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version     = '2.1.0'
+  s.version         = '3.0.0'
   s.licenses    = ['Apache License (2.0)']
   s.summary     = "Poll HTTP endpoints with Logstash."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -18,14 +18,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-mixin-http_client', ">= 2.2.4", "< 3.0.0"
   s.add_runtime_dependency 'stud', "~> 0.0.22"
-  s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
 
   s.add_development_dependency 'logstash-codec-json'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
-  s.add_development_dependency 'timecop'
 end

data/spec/inputs/http_poller_spec.rb

@@ -1,14 +1,11 @@
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/inputs/http_poller'
 require 'flores/random'
-require "timecop"
 
 describe LogStash::Inputs::HTTP_Poller do
   let(:metadata_target) { "_http_poller_metadata" }
   let(:queue) { Queue.new }
-  let(:default_schedule) {
-    { "cron" => "* * * * * UTC" }
-  }
+  let(:default_interval) { 5 }
   let(:default_name) { "url1 " }
   let(:default_url) { "http://localhost:1827" }
   let(:default_urls) {
@@ -18,7 +15,7 @@ describe LogStash::Inputs::HTTP_Poller do
   }
   let(:default_opts) {
     {
-      "schedule" => default_schedule,
+      "interval" => default_interval,
       "urls" => default_urls,
       "codec" => "json",
       "metadata_target" => metadata_target
@@ -34,13 +31,9 @@ describe LogStash::Inputs::HTTP_Poller do
     end
 
     describe "#run" do
-      it "should setup a scheduler" do
-        runner = Thread.new do
-          subject.run(double("queue"))
-          expect(subject.instance_variable_get("@scheduler")).to be_a_kind_of(Rufus::Scheduler)
-        end
-        runner.kill
-        runner.join
+      it "should run at the specified interval" do
+        expect(Stud).to receive(:interval).with(default_interval).once
+        subject.run(double("queue"))
       end
     end
 
@@ -158,166 +151,9 @@ describe LogStash::Inputs::HTTP_Poller do
     end
   end
 
-  describe "scheduler configuration" do
-    context "given an interval" do
-      let(:opts) {
-        {
-          "interval" => 2,
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should run once in each interval" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        #T       0123456
-        #events  x x x x
-        #expects 3 events at T=5
-        sleep 5
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(3)
-      end
-    end
-
-    context "given both interval and schedule options" do
-      let(:opts) {
-        {
-          "interval" => 1,
-          "schedule" => { "every" => "5s" },
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should raise ConfigurationError" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          expect{instance.run(queue)}.to raise_error(LogStash::ConfigurationError)
-        end
-        instance.stop
-        runner.kill
-        runner.join
-      end
-    end
-
-    context "given 'cron' expression" do
-      let(:opts) {
-        {
-          "schedule" => { "cron" => "* * * * * UTC" },
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
-        Timecop.scale(60)
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        sleep 3
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(2)
-        Timecop.return
-      end
-    end
-
-    context "given 'at' expression" do
-      let(:opts) {
-        {
-          "schedule" => { "at" => "2000-01-01 00:05:00 +0000"},
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        Timecop.travel(Time.new(2000,1,1,0,0,0,'+00:00'))
-        Timecop.scale(60 * 5)
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        sleep 2
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(1)
-        Timecop.return
-      end
-    end
-
-    context "given 'every' expression" do
-      let(:opts) {
-        {
-          "schedule" => { "every" => "2s"},
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        #T       0123456
-        #events  x x x x
-        #expects 3 events at T=5
-        sleep 5
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(3)
-      end
-    end
-
-    context "given 'in' expression" do
-      let(:opts) {
-        {
-          "schedule" => { "in" => "2s"},
-          "urls" => default_urls,
-          "codec" => "json",
-          "metadata_target" => metadata_target
-        }
-      }
-      it "should run at the schedule" do
-        instance = klass.new(opts)
-        instance.register
-        queue = Queue.new
-        runner = Thread.new do
-          instance.run(queue)
-        end
-        sleep 3
-        instance.stop
-        runner.kill
-        runner.join
-        expect(queue.size).to eq(1)
-      end
-    end
-  end
-
   describe "events" do
     shared_examples("matching metadata") {
-      let(:metadata) { event[metadata_target] }
+      let(:metadata) { event.get(metadata_target) }
 
       it "should have the correct name" do
         expect(metadata["name"]).to eql(name)
@@ -355,11 +191,11 @@ describe LogStash::Inputs::HTTP_Poller do
       end
 
       it "should enqueue a message with 'http_request_failure' set" do
-        expect(event["http_request_failure"]).to be_a(Hash)
+        expect(event.get("http_request_failure")).to be_a(Hash)
       end
 
       it "should tag the event with '_http_request_failure'" do
-        expect(event["tags"]).to include('_http_request_failure')
+        expect(event.get("tags")).to include('_http_request_failure')
       end
 
       it "should invoke handle failure exactly once" do
@@ -440,7 +276,7 @@ describe LogStash::Inputs::HTTP_Poller do
 
         it "should not have any metadata on the event" do
           instance.send(:run_once, queue)
-          expect(event[metadata_target]).to be_nil
+          expect(event.get(metadata_target)).to be_nil
         end
       end
 
@@ -456,9 +292,7 @@ describe LogStash::Inputs::HTTP_Poller do
         }
         let(:opts) {
           {
-            "schedule" => {
-              "cron" => "* * * * * UTC"
-              },
+            "interval" => default_interval,
             "urls" => {
               default_name => url
             },
@@ -482,7 +316,7 @@ describe LogStash::Inputs::HTTP_Poller do
           # When events go through the pipeline they are java-ified
           # this normalizes the payload to java types
           payload_normalized = LogStash::Json.load(LogStash::Json.dump(payload))
-          expect(event[target]).to include(payload_normalized)
+          expect(event.get(target)).to include(payload_normalized)
         end
       end
     end

metadata

@@ -1,45 +1,46 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
 - andrewvc
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
-  prerelease: false
+        version: '2.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-http_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -48,9 +49,8 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-mixin-http_client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -60,90 +60,64 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.0.0
 - !ruby/object:Gem::Dependency
+  name: stud
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.0.22
-  name: stud
-  prerelease: false
   type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.0.22
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.0.9
-  name: rufus-scheduler
   prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.0.9
+        version: 0.0.22
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
   name: logstash-codec-json
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: flores
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: flores
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: timecop
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -163,7 +137,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: input
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -178,9 +152,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: Poll HTTP endpoints with Logstash.
 test_files: