logstash-input-blueliv 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c63d09ff15c1485740cd2f3ac3044e46caa5d060
-  data.tar.gz: 16d60dfe1583afcf34430d281cca0fd096eeb57c
+  metadata.gz: 239a66eaf18c2e97d86c305e8150ea1c3600656e
+  data.tar.gz: d463808c18f1980f338dc13c4308c2ff4942e79a
 SHA512:
-  metadata.gz: b9632f2ff75ef80e39fcb0fdf21fc1cf9734de2ae8dcbb4f2cc8639f2f0c86f647eaf5cc0588d600a607e69ef21c1fb1dac86c11836082e75e7fbcb46046785f
-  data.tar.gz: 8e94b40686619ba9cac30f4dc359503e8e47149d98b769d0707571f9d78377ea6f5e09d891b1b51e5fc47cc4c92bf9de258617a8e66b32fa878063bfa4db6c8b
+  metadata.gz: 287345891c6a327df5d89630b95f05d425d4e25c3f54538f1355c017e587915a372cb446aef990fcd758bf8b36c8ee1880153d4d786b9624f4023cd16148fdfc
+  data.tar.gz: 8478591f6323e46467bf516ac5dae592f013499b2cd112ca113e19680ea9227916263c8f88d9c8eb92faab4ebc2dbec1e33a800f057edff02ee593720124d2e1

data/README.md

@@ -6,12 +6,19 @@ This is an input plugin for [Logstash](https://github.com/elasticsearch/logstash
 
 * API key (get yours <a href="https://map.blueliv.com" target="_blank">here</a>)
 * Logstash >= 1.5.0
-* 1.5 GB of RAM of heap size for Logstash (``-Xmx1500m``)
+* ElasticSearch >= 2.0.0 (Tested on 2.4.0)
 
 ## Installing
 
 ```
-$LS_HOME/bin/plugin install logstash-input-blueliv
+# In logstash version < 2.3
+
+$LS_HOME/bin/plugin install --version 1.1.0 logstash-input-blueliv
+
+# In logstash version >2.3
+$LS_HOME/bin/logstash-plugin install logstash-input-blueliv
+
+
 ```
 
 ### Configuration
@@ -23,21 +30,33 @@ This plugin has the following configuration parameters:
 + ``http_timeout``(default: ``500`` seconds): HTTP timeout for each API call.
 + ``feeds``: It is a [hash](http://ruby-doc.org/core-1.9.3/Hash.html) that specifies the parameters to access each one of our feeds. Each feed may be configured with the following properties:
     + ``active`` (default: ``false``): if the feed is active or not.
-    + ``feed_type`` (default: ``test``): the type of the feed that you want. For **Crime Servers** apart from ``test`` (for _debug_ purposes) you only have ``all``. As of **Bot IPs** you may choose between ``non_pos`` (all BotIPs **but** the ones from Point-of-Sale), ``pos`` (only from POS)  or ``full`` (all of them) feed.
+    + ``feed_type`` (default: ``test``): the type of the feed that you want. For **Crime Servers** apart from ``test`` (for _debug_ purposes) you have ``recent`` (1 hour updates) and ``last`` (15 minutes updates). As of **Bot IPs** you may choose between ``non_pos`` (all BotIPs **but** the ones from Point-of-Sale), ``pos`` (only from POS)  or ``full`` (all of them) feed.
     + ``interval`` (default: ``600`` seconds for BotIPs and ``900`` seconds for Crime Servers). The intervall of polling data from our API.
 
 The default configuration for ``feeds`` field is the following:
 ```javascript
-"crimeservers" => {
+{
+  "attacks" => {
     "active" => false,
-    "feed_type" => "test",
-    "interval" => 900,
+    "feed_type" => "recent",
+    "interval" => 600
   },
   "botips" => {
     "active" => false,
     "feed_type" => "test",
     "interval" => 600
+  },
+  "crimeservers" => {
+    "active" => true,
+    "feed_type" => "test",
+    "interval" => 900
+  },
+  "malwares" => {
+    "active" => false,
+    "feed_type" => "recent",
+    "interval" => 3600
   }
+}
 ```
 
 
@@ -48,18 +67,26 @@ input {
  blueliv {
   api_key => "<YOUR API KEY>"
   feeds => {
+    "attacks" => {
+      "active" => "true"
+      "feed_type" => "recent"
+    }
     "botips" => {
-      "active" => true
+      "active" => "true"
       "feed_type" => "non_pos"
     }
     "crimeservers" => {
-      "active" => true
-      "feed_type" => "all"
+      "active" => "true"
+      "feed_type" => "recent"
+    }
+    "malwares" => {
+      "active" => "true"
+      "feed_type" => "recent"
     }
   }
- }
 }
 ```
+
 Be aware that if you do not specify a given field, the default value will be configured. In this case, we did not touch the ``interval`` field for the feeds, so the defaults will apply.
 
 ## Need Help?

data/lib/logstash/inputs/blueliv.rb

@@ -8,7 +8,7 @@ require "rest-client"
 require "securerandom"
 
 
-USER_AGENT = "Logstash v0.1.2"
+USER_AGENT = "Logstash 1.1.0"
 API_CLIENT = "6ee37a93-d064-464b-b4c1-c37e9656273f"
 
 RESOURCES = {
@@ -32,35 +32,69 @@ RESOURCES = {
     :endpoint => "/v1/ip",
     :feeds => {
       :non_pos => {
-        600 => "/recent",
-        3600 => "/last"
+        600 => "/last",
+        3600 => "/recent"
       },
       :pos => {
-        600 => "/pos/recent",
-        3600 => "/pos/last"
+        600 => "/pos/last",
+        3600 => "/pos/recent"
       },
       :full => {
-        600 => "/full/recent",
-        3600 => "/full/last"
+        600 => "/full/last",
+        3600 => "/full/recent"
       },
       :test => {
         600 => "/test"
       }
     }
+  },
+  :attacks => {
+    :items => "attacks",
+    :endpoint => "/v1/attack",
+    :feeds => {
+      :last => {
+        1800 => "/last"
+      },
+      :recent => {
+        10800 => "/recent"
+      }
+    }
+  },
+  :malwares => {
+    :items => "malwares",
+    :endpoint => "/v1/malware",
+    :feeds => {
+      :last => {
+        600 => "/last"
+      },
+      :recent => {
+        3600 => "/recent"
+      }
+    }
   }
 }
 
 DEFAULT_CONFIG = {
+  "attacks" => {
+    "active" => false,
+    "feed_type" => "recent",
+    "interval" => 600
+  },
+  "botips" => {
+    "active" => false,
+    "feed_type" => "test",
+    "interval" => 600
+  },
   "crimeservers" => {
-      "active" => true,
-      "feed_type" => "test",
-      "interval" => 900
-    },
-    "botips" => {
-      "active" => false,
-      "feed_type" => "test",
-      "interval" => 600
-    }
+    "active" => true,
+    "feed_type" => "test",
+    "interval" => 900
+  },
+  "malwares" => {
+    "active" => false,
+    "feed_type" => "recent",
+    "interval" => 3600
+  }
 }
 
 INITIALIZE_FILE = "blueliv.ini"
@@ -93,8 +127,8 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
   def run(queue)
       threads = []
       @feeds.each do |name, conf|
-        if feeds[name]["active"] == 'true'
-          url, interval = get_url(name, @feeds[name]["feed_type"], @feeds[name]["interval"])
+        if conf["active"] == 'true'
+          url, interval = get_url(name, conf["feed_type"], conf["interval"])
           threads << Thread.new{get_feed_each(queue, name, url, interval)}
         end
       end
@@ -165,10 +199,11 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
         response = client.get("#{url}?key=#{API_CLIENT}", :Authorization => @auth, :timeout => @timeout,
           :user_agent => USER_AGENT, :headers => {"X-API-CLIENT" => API_CLIENT})
         response_json = JSON.parse(response.body)
-        items = response_json[RESOURCES[name.to_sym][:items]]
+        keyItems = RESOURCES[name.to_sym][:items]
+        items = response_json[keyItems]
         items.each do |it|
-          it["location"] = [it["longitude"].to_f, it["latitude"].to_f]
-          collection = RESOURCES[name.to_sym][:items].downcase
+          collection = keyItems.downcase
+          it = send("map_"+collection, it)
           it["@collection"] = collection
           it["document_id"] = if it.has_key?("_id") then it["_id"] else SecureRandom.base64(32) end
           it.delete("_id") if it.has_key?("_id")
@@ -198,6 +233,7 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
         end
       rescue Exception => e
         @logger.info("Will retry in #{FAILURE_SLEEP} seconds")
+        @logger.info(e.message+"\n"+e.backtrace.join("\n"))
         sleep(FAILURE_SLEEP)
       end
     end
@@ -207,4 +243,35 @@ class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
     RestClient
   end
 
+  private
+  def map_global_location(it)
+    it["location"] = [it["longitude"].to_f, it["latitude"].to_f]
+    return it
+  end
+
+  def map_ips(it)
+    map_global_location(it)
+  end
+
+  def map_crimeservers(it)
+    map_global_location(it)
+  end
+
+  def map_attacks(it)
+    source = it["source"];
+    #map_global_location(source);
+    it["location"] = [source["longitude"].to_f, source["latitude"].to_f]
+    firstEvent = DateTime.parse(it["firstEvent"])
+    lastEvent = DateTime.parse(it["lastEvent"])
+    duration = ((lastEvent - firstEvent)* 24 * 60 * 60).to_i
+    if duration < 0 
+      duration = -duration
+    end
+    it[:duration] = duration
+    return it
+  end
+  def map_malwares(it)
+    it["_id"] = it["sha256"]
+    return it
+  end
 end

data/logstash-input-blueliv.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = "logstash-input-blueliv"
-  s.version = "1.0.1"
+  s.version = "1.1.0"
   s.licenses = ["Apache License (2.0)"]
   s.summary = "This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program"

metadata

@@ -1,78 +1,78 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-blueliv
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Blueliv
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-19 00:00:00.000000000 Z
+date: 2017-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 1.4.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
+  name: logstash-core
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.4.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-codec-plain
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: rest-client
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: '0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.8.0
+  name: rest-client
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: logstash-devutils
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program
 email: community@blueliv.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGELOG.md
 - Gemfile
 - LICENSE
@@ -93,17 +93,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds.