logstash-input-signalsciences 0.3.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef2ac72345f4918d2c16f7d0a72e256a8ec06ce4
-  data.tar.gz: 6999e82e6d7e4d86b130fb971c93bcaa2dffc645
+  metadata.gz: 49e1f0ae15ed2dc2b8d8fb079c3d6de416b3d7fc
+  data.tar.gz: 011cc2201fc62fea29b465e22f73b5203205e055
 SHA512:
-  metadata.gz: 7add1bc93c67072363afac43d244840b5743b8ba4d9b9bd3b0471fb0e89efa403a128e0b5462398f6925b1f72706059b891c6b582d85750b79961db2b9760e6d
-  data.tar.gz: e4ddf8193986d9637e812f5b5c35439227fae81886c50a3433f7d6f496dd2ffc6dc8f9ab2a7261dc0b12788f5ac22d2b9b52433c90cae7986529d1578bea5fe4
+  metadata.gz: f6a2fb3ac45c9b8435bea206baa4b40306deaa069dd471cdb7aeadba859cab51a25fc89dcfc7bb74d4718ef932b71b73ba1e0f28a7943e62993b21436ddc83ce
+  data.tar.gz: c2c8c28854869195dcb9ec9a02b2c1bb08af6307cbd105149f1cf089181f35011e28d659b2aa6127835fcb16662b168990425b8320cc0946a64c53c480b74e00

data/lib/logstash/inputs/signalsciences.rb

@@ -19,7 +19,9 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
   # Signal Sciences API account username.
   config :email, :validate => :string, :default => "nobody@signalsciences.com"
   # Signal Sciences API password.
-  config :password, :validate => :string, :default => "nobody"
+  config :password, :validate => :string, :default => ""
+  # Signal Sciences API access token.
+  config :token, :validate => :string, :default => ""
   # Corp and site to pull data from.
   config :corp, :validate => :string, :default => "not_provided"
   config :site, :validate => :string, :default => "not_provided"
@@ -37,9 +39,15 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
     @http = Net::HTTP.new('dashboard.signalsciences.net', 443)
     @http.set_debug_output($stdout) if @debug
     @login = Net::HTTP::Post.new("/api/v0/auth")
+    @auth_mode = "password"
     @get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests")
     @http.use_ssl = true
 
+    # determine authentication mode
+    if @token != ""
+      @auth_mode = "token"
+    end
+  
     # check if from value is less than 1 min
     if @from < 60
       @logger.warn("from value is less than 1 min, increasing from value to 1 minute.")
@@ -56,7 +64,7 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
     @interval = @from
 
     # set version for UA string
-    @version = "0.3.1"
+    @version = "1.0.0"
 
     @logger.info("Fetching Signal Sciences request data every #{@interval / 60} minutes.")
   end # def register
@@ -64,8 +72,10 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
   def run(queue)
     # we can abort the loop if stop? becomes true
     while !stop?
-      @login['User-Agent'] = "logstash-signalsciences/#{@version}"
-      @login.body = URI.encode_www_form({"email" => @email, "password" => @password})
+      if @auth_mode == "password"
+        @login['User-Agent'] = "logstash-signalsciences/#{@version}"
+        @login.body = URI.encode_www_form({"email" => @email, "password" => @password})
+      end
 
       if fetch(queue)
         @logger.info("Requests feed retreived successfully.")
@@ -84,112 +94,142 @@ class LogStash::Inputs::Signalsciences < LogStash::Inputs::Base
   end # def run
 
   def fetch(queue)
-    begin
-      response = @http.request(@login)
-      @logger.warn("login response: #{response.code}")
-    rescue
-      @logger.warn("Could not reach API endpoint to login!")
-      return false
+    if @auth_mode == "password"
+      begin
+        response = @http.request(@login)
+        @logger.warn("login response: #{response.code}")
+      rescue
+        @logger.warn("Could not reach API endpoint to login!")
+        return false
+      end
+
+      json = JSON.parse(response.body)
+
+      if json.has_key? "message"
+        # failed to login
+        @logger.warn("login: #{json['message']}")
+        return false
+      end
+
+      bearer_token = json['token']
     end
 
-    json = JSON.parse(response.body)
+    
+    # Both the from and until parameters must fall on full minute boundaries,
+    # see https://docs.signalsciences.net/faq/extract-your-data/.
+    t = Time.now.utc.strftime("%Y-%m-%d %H:%M:0")
+    dt = DateTime.parse(t)
+    timestamp_until = dt.to_time.to_i - 300 # now - 5 minutes
+    timestamp_from = (timestamp_until - @from) # @until - @from
+
+    if @debug
+      hfrom = Time.at(timestamp_from).to_datetime
+      huntil = Time.at(timestamp_until).to_datetime
+      @logger.info("From #{hfrom} Until #{huntil}")
+    end
 
-    if json.has_key? "message"
-      # failed to login
-      @logger.warn("login: #{json['message']}")
-      return false
+    # Set up iniital get request and initial next_uri
+    @logger.info("Requesting data: /api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
+    get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
+    next_uri = "not empty on first pass"
 
-    else
-      token = json['token']
-      # Both the from and until parameters must fall on full minute boundaries,
-      # see https://docs.signalsciences.net/faq/extract-your-data/.
-      t = Time.now.utc.strftime("%Y-%m-%d %H:%M:0")
-      dt = DateTime.parse(t)
-      timestamp_until = dt.to_time.to_i - 300 # now - 5 minutes
-      timestamp_from = (timestamp_until - @from) # @until - @from
+    # Loop through results until next_uri is empty.
+    while !next_uri.empty?
+      if @auth_mode == "password"
+        get["Authorization"] = "Bearer #{bearer_token}"
+      else
+        get["x-api-user"] = @email
+        get["x-api-token"] = @token
+      end
+      get['User-Agent'] = "logstash-signalsciences/#{@version}"
 
-      if @debug
-        hfrom = Time.at(timestamp_from).to_datetime
-        huntil = Time.at(timestamp_until).to_datetime
-        @logger.info("From #{hfrom} Until #{huntil}")
+      begin
+        response = @http.request(get)
+      rescue
+        @logger.warn("Could not reach API endpoint to retreive reqeusts feed!")
+        return false
+      end
+
+      if response.code == "524"
+        @logger.warn("524 - Origin Timeout!")
+        @logger.info("Another attempt will be made later.")
+        return false
+      end
+
+      if response.code == "429"
+        @logger.warn("429 - Too Many Requests!")
+        @logger.info("API request throttling as been triggered, another attempt will be made later. Contact support if this error continues.")
+        return false
+      end
+
+      if response.code == "404"
+        @logger.warn("404 - Not Found!")
+        return false
       end
 
-      # Set up iniital get request and initial next_uri
-      @logger.info("Requesting data: /api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
-      get = Net::HTTP::Get.new("/api/v0/corps/#{@corp}/sites/#{@site}/feed/requests?from=#{timestamp_from}&until=#{timestamp_until}")
-      next_uri = "not empty on first pass"
+      if response.code == "401"
+        @logger.warn("401 - Unauthorized!")
+        return false
+      end
+
+      json = JSON.parse(response.body)
 
-      # Loop through results until next_uri is empty.
-      while !next_uri.empty?
-        get["Authorization"] = "Bearer #{token}"
-        get['User-Agent'] = "logstash-signalsciences/#{@version}"
+      #check for message, error, e.g. missing query string parameter
+      if json.has_key? "message"
+        # some error occured, report it.
+        @logger.warn("Error accessing API (auth mode: #{@auth_mode}), status code: #{response.code} with message: #{json['message']}")
+        return false
+      end
 
+      # log json payloads
+      json['data'].each do |payload|
+
+        # explode headersIn out to headerIn entries
+        temp = {}
         begin
-          response = @http.request(get)
-        rescue
-          @logger.warn("Could not reach API endpoint to retreive reqeusts feed!")
-          return false
-        end
-        json = JSON.parse(response.body)
-
-        #check for message, error, e.g. missing query string parameter
-        if json.has_key? "message"
-          # some error occured, report it.
-          @logger.warn("Error accessing API (#{token}), status code: #{response.code} with message: #{json['message']}")
-          return false
-
-        else
-          # log json payloads
-          json['data'].each do |payload|
-
-            # explode headersIn out to headerIn entries
-            temp = {}
-            begin
-              payload['headersIn'].each { |k,v| temp[k] = v }
-              payload["headerIn"] = temp
-            rescue NoMethodError
-              if @debug
-                @logger.debug("payload['headersIn'] is empty for id #{payload['id']}, skipping append.")
-              end
-            end
-
-            # explode headersOut out to headerOut entries
-            temp = {}
-            begin
-              payload['headersOut'].each { |k,v| temp[k] = v }
-              payload["headerOut"] = temp
-            rescue NoMethodError
-              if @debug
-                @logger.info("payload['headersOut'] is empty for id #{payload['id']}, skipping append.")
-              end
-            end
-
-            # explode tags out to tag entries
-            temp = {}
-            payload['tags'].each do |x|
-              temp[x['type']] = x
-            end
-            payload['tag'] = temp
-
-            # add the event
-            
-            event = LogStash::Event.new("message" => payload, "host" => @host)
-
-            decorate(event)
-            queue << event
+          payload['headersIn'].each { |k,v| temp[k] = v }
+          payload["headerIn"] = temp
+        rescue NoMethodError
+          if @debug
+            @logger.debug("payload['headersIn'] is empty for id #{payload['id']}, skipping append.")
           end
+        end
 
-          # get the next uri value
-          next_uri = json['next']['uri']
+        # explode headersOut out to headerOut entries
+        temp = {}
+        begin
+          payload['headersOut'].each { |k,v| temp[k] = v }
+          payload["headerOut"] = temp
+        rescue NoMethodError
           if @debug
-            logger.info("Next URI: #{next_uri}")
+            @logger.info("payload['headersOut'] is empty for id #{payload['id']}, skipping append.")
           end
+        end
 
-          # continue retreiving next_uri if it exists
-          if !next_uri.empty?
-            get = Net::HTTP::Get.new(next_uri)
-          end
+        # explode tags out to tag entries
+        temp = {}
+        payload['tags'].each do |x|
+          temp[x['type']] = x
         end
+        payload['tag'] = temp
+
+        # add the event
+        
+        event = LogStash::Event.new("message" => payload, "host" => @host)
+
+        decorate(event)
+        queue << event
+      end
+
+      # get the next uri value
+      next_uri = json['next']['uri']
+      if @debug
+        logger.info("Next URI: #{next_uri}")
+      end
+
+      # continue retreiving next_uri if it exists
+      if !next_uri.empty?
+        get = Net::HTTP::Get.new(next_uri)
       end
     end
 

data/logstash-input-signalsciences.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-signalsciences'
-  s.version       = '0.3.1'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Logstash input plugin for Signal Sciences.'
   s.description   = 'Logstash input plugin for the Signal Sciences request feed endpoint https://docs.signalsciences.net/api/#get-request-feed'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-signalsciences
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 1.0.0
 platform: ruby
 authors:
 - foospidy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-03 00:00:00.000000000 Z
+date: 2020-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api