logstash-input-signalsciences 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7677c8fd764c77486906ebebb98b7d95a5d5d557
-  data.tar.gz: dc4601ef183dde9d4b8d4014d6cd0a06732b6b09
+  metadata.gz: 62c7c90326f3b048fd6059c9b97067b8033ec918
+  data.tar.gz: 0fd2c6793585ba90130075bfad296a01339f1611
 SHA512:
-  metadata.gz: 20b7b7971d2f708fbfa0f866f80851770eba97c77e00176135ea4f1b2de098d598d587b3254cc4ed93ac2076aa37f26575c819d8176e3360c0623e9305774101
-  data.tar.gz: 7fccc04bca6c78d89918f8b16c1fb456c0c5d66ecc9ee1d5e993d301e1e892a991d815ef64537c9306fb92c3ad96cfb5c98eb3b2546607ab534066b5df32d714
+  metadata.gz: cc4c953e8386b244b09d6797f814036ed410442456269292ec7dad0ad5652ea0294ef089071d76a664dcaaf2cbe1dee1fed1440848db75e8d445545c5a7be911
+  data.tar.gz: 500b60c32af0408b6e51e87ef0d2346e6aaeace213b447e5a38e4576a7d7a8611e380ed41a3019ea0868a69eee7961fcde680867304c2471c385cf0954cdddc8

data/CHANGELOG.md

@@ -1,5 +1,14 @@
+## 0.2.0
+
+- Changed time to UTC
+- Changed default from value to 5 min
+- Updated from value to allow for pulling data every 1 min
+- Added additional logging for debug
+
 ## 0.1.1
-  - Fixed references to vulnerable dependencieas in Gemfile.lock
+
+- Fixed references to vulnerable dependencieas in Gemfile.lock
 
 ## 0.1.0
-  - Plugin created with the logstash plugin generator
+
+- Plugin created with the logstash plugin generator

data/INSTRUCTIONS.md

@@ -8,7 +8,7 @@ There are two methods of installing this plugin.
 
 1. The plugin is published on rubygems.org (https://rubygems.org/gems/logstash-input-signalsciences) so it can be easily installed via Logstash by using the following command:
 
-`logstash-plugin install logstash-input-signalciences`
+`logstash-plugin install logstash-input-signalsciences`
 
 2. You can build and install the gem file locally by using the `Makefile` on your system. However, you will need to ensure Ruby requirements for building gem files are met. Use the following commands:
 
@@ -29,7 +29,7 @@ you can also refer to Elasticsearch's getting started page: https://www.elastic.
 
 1. The first requirement is Java 8, run:
 
-```
+```shell
 brew update
 brew tap caskroom/versions
 brew cask install java8
@@ -39,7 +39,7 @@ Reference https://stackoverflow.com/questions/24342886/how-to-install-java-8-on-
 
 2. Next install Elasticsearch, Logstash, and Kibana, run:
 
-```
+```shell
 brew install elasticsearch
 brew install logstash
 brew install kibana
@@ -47,7 +47,7 @@ brew install kibana
 
 With brew you can start/stop these services by running:
 
-```
+```shell
 brew services start elasticsearch
 brew services start logstash
 brew services start kibana
@@ -67,4 +67,4 @@ If you want to create a new plugin, see:
 
 1. Plugin generator https://www.elastic.co/guide/en/logstash/current/plugins-inputs-generator.html, usage: https://github.com/elastic/logstash/blob/master/docs/static/plugin-generator.asciidoc
 
-2. Submitting your plugin to [RubyGems.org and the Logstash-plugin repository] (https://www.elastic.co/guide/en/logstash/current/submitting-plugin.html).
+2. Submitting your plugin to [RubyGems.org and the Logstash-plugin repository](https://www.elastic.co/guide/en/logstash/current/submitting-plugin.html).

data/README.md

@@ -1,5 +1,7 @@
 # Logstash Input Plugin for Signal Sciences
 
+For more detail on using the Signal Sciences logstash plugin, see [instructions](INSTRUCTIONS.md).
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/inputs/signalsciences.rb

@@ -25,12 +25,9 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
   config :site, :validate => :string, :default => "not_provided"
   # Number of seconds in the past to filter data on
   # This value will also be used to set the interval at which the API is polled.
-  config :from, :validate => :number, :default => 600
+  config :from, :validate => :number, :default => 300
   # Debug for plugin development.
   config :debug, :validate => :boolean, :default => false
-
-  # Set how frequently messages should be sent.
-  #
   # The default, `600`, means fetch data every 10 minutes.
   config :interval, :validate => :number, :default => 600
 
@@ -43,10 +40,10 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
     @get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests")
     @http.use_ssl = true
 
-    # check if from value is 5 minutes or less
-    if @from <= 300
-      @logger.warn("from value is 5 minutes or less, increasing from value to 10 minutes.")
-      @from = 600
+    # check if from value is less than 1 min
+    if @from < 60
+      @logger.warn("from value is less than 1 min, increasing from value to 1 minute.")
+      @from = 60
     end
 
     # check if from value is greater than 24 hours
@@ -55,10 +52,10 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
       @from = 86400
     end
 
-    # set interval to value of from @from minus one minute
-    interval = @from
+    # set interval to value of from @from minus five minutes
+    @interval = @from
 
-    @logger.info("Fetching Signal Sciences request data every #{interval / 60} minutes.")
+    @logger.info("Fetching Signal Sciences request data every #{@interval / 60} minutes.")
   end # def register
 
   def run(queue)
@@ -68,6 +65,8 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
 
       if fetch(queue)
         @logger.info("Requests feed retreived successfully.")
+      else
+        @logger.warn("Problem retreiving request!")
       end
 
       # because the sleep interval can be big, when shutdown happens
@@ -75,6 +74,7 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
       # Stud.stoppable_sleep will frequently evaluate the given block
       # and abort the sleep(@interval) if the return value is true
       #Stud.stoppable_sleep(@interval) { stop? }
+      @logger.info("Sleep #{@interval}")
       Stud.stoppable_sleep(@interval) { stop? }
     end # loop
   end # def run
@@ -82,6 +82,7 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
   def fetch(queue)
     begin
       response = @http.request(@login)
+      @logger.warn("login response: #{response.code}")
     rescue
       @logger.warn("Could not reach API endpoint to login!")
       return false
@@ -98,12 +99,19 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
       token = json['token']
       # Both the from and until parameters must fall on full minute boundaries,
       # see https://docs.signalsciences.net/faq/extract-your-data/.
-      t = Time.now.strftime("%Y-%m-%d %H:%M:0")
+      t = Time.now.utc.strftime("%Y-%m-%d %H:%M:0")
       dt = DateTime.parse(t)
       timestamp_until = dt.to_time.to_i - 300 # now - 5 minutes
-      timestamp_from = (timestamp_until - @from) - 300 # @from - 5 minutes
+      timestamp_from = (timestamp_until - @from) # @until - @from
+
+      if @debug
+        hfrom = Time.at(timestamp_from).to_datetime
+        huntil = Time.at(timestamp_until).to_datetime
+        @logger.info("From #{hfrom} Until #{huntil}")
+      end
 
       # Set up iniital get request and initial next_uri
+      @logger.info("Requesting data: /api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
       get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
       next_uri = "not empty on first pass"
 
@@ -122,7 +130,7 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
         #check for message, error, e.g. missing query string parameter
         if json.has_key? "message"
           # some error occured, report it.
-          @logger.warn("get: #{json['message']} #{token}")
+          @logger.warn("Error accessing API (#{token}), status code: #{response.code} with message: #{json['message']}")
           return false
 
         else
@@ -135,7 +143,9 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
               payload['headersIn'].each { |k,v| temp[k] = v }
               payload["headerIn"] = temp
             rescue NoMethodError
-              @logger.info("payload['headersIn'] is empty for id #{payload['id']}, skipping append.")
+              if @debug
+                @logger.debug("payload['headersIn'] is empty for id #{payload['id']}, skipping append.")
+              end
             end
 
             # explode headersOut out to headerOut entries
@@ -144,7 +154,9 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
               payload['headersOut'].each { |k,v| temp[k] = v }
               payload["headerOut"] = temp
             rescue NoMethodError
-              @logger.info("payload['headersOut'] is empty for id #{payload['id']}, skipping append.")
+              if @debug
+                @logger.info("payload['headersOut'] is empty for id #{payload['id']}, skipping append.")
+              end
             end
 
             # explode tags out to tag entries
@@ -155,6 +167,7 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
             payload['tag'] = temp
 
             # add the event
+            
             event = LogStash::Event.new("message" => payload, "host" => @host)
 
             decorate(event)
@@ -163,6 +176,9 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
 
           # get the next uri value
           next_uri = json['next']['uri']
+          if @debug
+            logger.info("Next URI: #{next_uri}")
+          end
 
           # continue retreiving next_uri if it exists
           if !next_uri.empty?

data/logstash-input-signalsciences.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-signalsciences'
-  s.version       = '0.1.1'
+  s.version       = '0.2.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Logstash input plugin for Signal Sciences.'
   s.description   = 'Logstash input plugin for the Signal Sciences request feed endpoint https://docs.signalsciences.net/api/#get-request-feed'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-signalsciences
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - foospidy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-25 00:00:00.000000000 Z
+date: 2020-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -101,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: Logstash input plugin for Signal Sciences.