logstash-input-http_poller 4.0.3 → 5.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a32f39048f5329848504f38c8c89cf5e997cdc9b
-  data.tar.gz: 6e9a910ca6dd641db1ce87e2b23f76d05249d128
+SHA256:
+  metadata.gz: 50ae088dbf54c3b101fdbb36f6743d9525c4664ef9dc592ba3d351580a8712a8
+  data.tar.gz: 0ea8ab2cfae1a16dd3faadf4a1af9ccac2b9e235d1ac0a83353ed21e95c70b78
 SHA512:
-  metadata.gz: 5b20a8cefd5691306d89cbb68c52105adc50675a3a9f95e6fc39abe46f307ca5591cb88c4628e934805c902b03383eafe9e9c0b309c64e3999c1418a783407e6
-  data.tar.gz: 332c3f3e13855246c5bc821096558c9a58a65824491e107b2307d5c0366c3e20bf71c86af9683039414b1e1ea90d0b2f8a6df434bd27e82319c43d976ca313cf
+  metadata.gz: 47457f51157fb74494d11b03567af00c7855f1824b020740c836977fd99f3b91a06bbde2724148205971c4ff02dc539580537e90088dd36a14e5daf09311d94d
+  data.tar.gz: 982dd17052f30f21f8a4d6d1e852260f0955d11f0d42eae0dcc003a3478522fb4cb4a8310e57db821238127aefbe728cd758154e689045dc7dbc221ca29f617b

data/CHANGELOG.md

@@ -1,8 +1,27 @@
+## 5.0.2
+ - [DOC]Expanded url option to include Manticore keys [#119](https://github.com/logstash-plugins/logstash-input-http_poller/pull/119)
+
+## 5.0.1
+ - Fixed minor doc and doc formatting issues [#107](https://github.com/logstash-plugins/logstash-input-http_poller/pull/107)
+
+## 5.0.0
+ - Removed obsolete field `interval`
+
+## 4.0.6
+  - Changed `schedule` entry to show that it is required
+  [#102](https://github.com/logstash-plugins/logstash-input-http_poller/pull/102)
+
+## 4.0.5
+  - Docs: Set the default_codec doc attribute.
+
+## 4.0.4
+  - Update gemspec summary
+
 ## 4.0.3
   - Docs: Remove row in overview table to fix build error
   
 ## 4.0.2
-  - Don't bleed URLs credentials on startup and on exception #82
+  - Don't bleed URLs credentials on startup and on exception [#82](https://github.com/logstash-plugins/logstash-input-http_poller/pull/82)
 
 ## 4.0.1
   - Fix some documentation issues

data/LICENSE

@@ -1,3 +1,4 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
@@ -178,7 +179,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2020 Elastic and contributors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -199,4 +200,3 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-

data/docs/index.asciidoc

@@ -1,5 +1,6 @@
 :plugin: http_poller
 :type: input
+:default_codec: json
 
 ///////////////////////////////////////////
 START - GENERATED VARIABLES, DO NOT EDIT!
@@ -94,8 +95,6 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
-| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|no
-| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-automatic_retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-client_cert>> |a valid filesystem path|No
@@ -108,18 +107,20 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-keystore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-metadata_target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-proxy>> |<<,>>|No
 | <<plugins-{type}s-{plugin}-request_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-retry_non_idempotent>> |<<boolean,boolean>>|No
-| <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-urls>> |<<hash,hash>>|Yes
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
 |=======================================================================
 
@@ -128,23 +129,6 @@ input plugins.
 
 &nbsp;
 
-[id="plugins-{type}s-{plugin}-user"]
-===== `user` 
-
-  * Value type is <<string,string>>
-  * There is no default value for this setting.
-
-Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
-If you set this you must also set the `password` option.
-
-[id="plugins-{type}s-{plugin}-password"]
-===== `password` 
-
-  * Value type is <<password,password>>
-  * There is no default value for this setting.
-
-Password to be used in conjunction with the username for HTTP authentication.
-
 [id="plugins-{type}s-{plugin}-automatic_retries"]
 ===== `automatic_retries` 
 
@@ -248,6 +232,14 @@ If you'd like to work with the request/response metadata.
 Set this value to the name of the field you'd like to store a nested
 hash of metadata.
 
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Password to be used in conjunction with <<plugins-{type}s-{plugin}-user>> for HTTP authentication.
+
 [id="plugins-{type}s-{plugin}-pool_max"]
 ===== `pool_max` 
 
@@ -282,10 +274,7 @@ If you'd like to use an HTTP proxy . This supports multiple configuration syntax
   * Value type is <<number,number>>
   * Default value is `60`
 
-This module makes it easy to add a very fully configured HTTP client to logstash
-based on [Manticore](https://github.com/cheald/manticore).
-For an example of its usage see https://github.com/logstash-plugins/logstash-input-http_poller
-Timeout (in seconds) for the entire request
+Timeout (in seconds) for the entire request.
 
 [id="plugins-{type}s-{plugin}-retry_non_idempotent"]
 ===== `retry_non_idempotent` 
@@ -359,7 +348,45 @@ Specify the truststore type here. One of `JKS` or `PKCS12`. Default is `JKS`
   * There is no default value for this setting.
 
 A Hash of urls in this format : `"name" => "url"`.
-The name and the url will be passed in the outputed event
+The name and the url will be passed in the outputted event.
+
+The values in urls can be either:
+
+* a string url (which will be issued as an HTTP GET).
+* a sub-hash containing many useful keys provided by the Manticore backend:
+** url: the String url
+** method: (optional) the HTTP method to use (defaults to GET)
+** user: (optional) the HTTP Basic Auth user. The user must be under 
+   an auth sub-hash for Manticore, but this plugin also accepts it either way.
+** password: (optional) the HTTP Basic Auth password. The password 
+   must be under an auth sub-hash for Manticore, but this plugin accepts it either way.
+** headers: a hash containing key-value pairs of headers.
+** body: a string (supported only on POST and PUT requests)
+** possibly other options mentioned in the 
+   https://www.rubydoc.info/github/cheald/manticore/Manticore/Client#http-instance_method[Manticore docs].
+   Note that Manticore options that are not explicitly documented above are not
+   thoroughly tested and therefore liable to break in unexpected ways if we
+   replace the backend.
+
+*Notes:*
+
+* Passwords specified as a part of `urls` are prone to exposure in plugin log output. 
+The plugin does not declare them as passwords, and therefore doesn't wrap them in
+leak-reducing wrappers as we do elsewhere.
+* We don't guarantee that boolean-type options like Manticore's `follow_redirects` are supported
+correctly. The strings `true` or `false` may get passed through, and in ruby any
+string is "truthy."
+* Our implementation of this plugin precludes the ability to specify auth[:eager]
+as anything other than true
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Username to use with HTTP authentication for ALL requests. Note that you can also set this per-URL.
+If you set this you must also set the <<plugins-{type}s-{plugin}-password>> option.
 
 [id="plugins-{type}s-{plugin}-validate_after_inactivity"]
 ===== `validate_after_inactivity` 
@@ -367,13 +394,18 @@ The name and the url will be passed in the outputed event
   * Value type is <<number,number>>
   * Default value is `200`
 
-How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
-# You may want to set this lower, possibly to 0 if you get connection errors regularly
-Quoting the Apache commons docs (this client is based Apache Commmons):
-'Defines period of inactivity in milliseconds after which persistent connections must be re-validated prior to being leased to the consumer. Non-positive value passed to this method disables connection validation. This check helps detect connections that have become stale (half-closed) while kept inactive in the pool.'
-See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
-
+How long to wait before checking for a stale connection to determine if a keepalive request is needed.
+Consider setting this value lower than the default, possibly to 0, if you get connection errors regularly.
 
+This client is based on Apache Commons' HTTP implementation. Here's how the
+https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[Apache
+Commons documentation] describes this option: "Defines period of inactivity in
+milliseconds after which persistent connections must be re-validated prior to
+being leased to the consumer. Non-positive value passed to this method disables
+connection validation. This check helps detect connections that have become
+stale (half-closed) while kept inactive in the pool."
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]
+
+:default_codec!:

data/lib/logstash/inputs/http_poller.rb

@@ -17,8 +17,6 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   # The name and the url will be passed in the outputed event
   config :urls, :validate => :hash, :required => true
 
-  config :interval, :validate => :number, :obsolete => "The interval options is obsolete. Use schedule instead"
-
   # Schedule of when to periodically poll from the urls
   # Format: A hash with
   #   + key: "cron" | "every" | "in" | "at"
@@ -165,7 +163,7 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     # If there is a usable response. HEAD requests are `nil` and empty get
     # responses come up as "" which will cause the codec to not yield anything
     if body && body.size > 0
-      @codec.decode(body) do |decoded|
+      decode_and_flush(@codec, body) do |decoded|
         event = @target ? LogStash::Event.new(@target => decoded.to_hash) : decoded
         handle_decoded_event(queue, name, request, response, event, execution_time)
       end
@@ -175,6 +173,12 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
     end
   end
 
+  private
+  def decode_and_flush(codec, body, &yielder)
+    codec.decode(body, &yielder)
+    codec.flush(&yielder)
+  end
+
   private
   def handle_decoded_event(queue, name, request, response, event, execution_time)
     apply_metadata(event, name, request, response, execution_time)

data/logstash-input-http_poller.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
   s.name        = 'logstash-input-http_poller'
-  s.version         = '4.0.3'
+  s.version         = '5.0.2'
   s.licenses    = ['Apache License (2.0)']
-  s.summary     = "Poll HTTP endpoints with Logstash."
+  s.summary     = "Decodes the output of an HTTP API into events"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors     = [ "Elastic", "andrewvc"]
   s.email       = 'info@elastic.co'
@@ -20,11 +20,12 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'logstash-mixin-http_client', ">= 6.0.0", "< 7.0.0"
+  s.add_runtime_dependency 'logstash-mixin-http_client', "~> 7"
   s.add_runtime_dependency 'stud', "~> 0.0.22"
   s.add_runtime_dependency 'rufus-scheduler', "~>3.0.9"
 
   s.add_development_dependency 'logstash-codec-json'
+  s.add_development_dependency 'logstash-codec-line'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'flores'
   s.add_development_dependency 'timecop'

data/spec/inputs/http_poller_spec.rb

@@ -1,4 +1,5 @@
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/devutils/rspec/shared_examples"
 require 'logstash/inputs/http_poller'
 require 'flores/random'
 require "timecop"
@@ -462,6 +463,40 @@ describe LogStash::Inputs::HTTP_Poller do
           expect(event.get(target)).to include(payload_normalized)
         end
       end
+
+      context 'using a line codec' do
+        let(:opts) do
+          default_opts.merge({"codec" => "line"})
+        end
+        subject(:events) do
+          [].tap do |events|
+            events << queue.pop until queue.empty?
+          end
+        end
+
+        context 'when response has a trailing newline' do
+          let(:response_body) { "one\ntwo\nthree\nfour\n" }
+          it 'emits all events' do
+            expect(events.size).to equal(4)
+            messages = events.map{|e| e.get('message')}
+            expect(messages).to include('one')
+            expect(messages).to include('two')
+            expect(messages).to include('three')
+            expect(messages).to include('four')
+          end
+        end
+        context 'when response has no trailing newline' do
+          let(:response_body) { "one\ntwo\nthree\nfour" }
+          it 'emits all events' do
+            expect(events.size).to equal(4)
+            messages = events.map{|e| e.get('message')}
+            expect(messages).to include('one')
+            expect(messages).to include('two')
+            expect(messages).to include('three')
+            expect(messages).to include('four')
+          end
+        end
+      end
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-http_poller
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 5.0.2
 platform: ruby
 authors:
 - Elastic
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-20 00:00:00.000000000 Z
+date: 2020-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -48,23 +48,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 6.0.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7'
   name: logstash-mixin-http_client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 6.0.0
-    - - "<"
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 7.0.0
+        version: '7'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -107,6 +101,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-line
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -149,7 +157,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -186,9 +196,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
-summary: Poll HTTP endpoints with Logstash.
+summary: Decodes the output of an HTTP API into events
 test_files:
 - spec/inputs/http_poller_spec.rb