logstash-input-blueliv 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f93593a5f0ac2f84de2133f030b010f9d6e9ddcc
+  data.tar.gz: 96f7fb2c72c1fbcce6bf03e59d0564084e5ef836
+SHA512:
+  metadata.gz: 94a6b4fa308af18e6e4a01618c7443fc697aa312197475f7ec5fe34f3b4ceb1b80add236292b907cc54f942530d1d6ef114e67218d5d105131f5367aea1944d8
+  data.tar.gz: 6ba937047471ea2ca1df09c2e27d1084ae0e25799acd83bcd57607c48d62f7ee5096a67f994023f02c2f7c685188474750c6b8ffecf3cac0c97540cd70593c9a

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+Gemfile.lock
+Gemfile.bak
+.bundle
+vendor
+*.json

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+**v1.0.0 - 2015 Jun 30**
+
++ First version

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2015 Blueliv
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,69 @@
+# Logstash Input Plugin by Blueliv
+
+This is an input plugin for [Logstash](https://github.com/elasticsearch/logstash) created and maintained by Blueliv (community@blueliv.com), that allows to access Blueliv's Cyber-Threat Intelligence feeds, such as Crime Servers and Bot IPs.
+
+## Minimum Requirements
+
+* API key (get yours <a href="https://map.blueliv.com" target="_blank">here</a>)
+* Logstash >= 1.5.0
+* 1.5 GB of RAM of heap size for Logstash (``-Xmx1500m``)
+
+## Installing
+
+```
+$LS_HOME/bin/plugin install logstash-input-blueliv
+```
+
+### Configuration
+
+This plugin has the following configuration parameters:
+
++ ``api_url`` (default: ``https://api.blueliv.com``): the URL of the API. It may change if you are using our free or commercial API.
++ ``api_key``: The API key to access our feeds. This parameter is **mandatory**.
++ ``http_timeout``(default: ``500`` seconds): HTTP timeout for each API call.
++ ``feeds``: It is a [hash](http://ruby-doc.org/core-1.9.3/Hash.html) that specifies the parameters to access each one of our feeds. Each feed may be configured with the following properties:
+    + ``active`` (default: ``false``): if the feed is active or not.
+    + ``feed_type`` (default: ``test``): the type of the feed that you want. For **Crime Servers** apart from ``test`` (for _debug_ purposes) you only have ``all``. As of **Bot IPs** you may choose between ``non_pos`` (all BotIPs **but** the ones from Point-of-Sale), ``pos`` (only from POS)  or ``full`` (all of them) feed.
+    + ``interval`` (default: ``600`` seconds for BotIPs and ``900`` seconds for Crime Servers). The intervall of polling data from our API.
+
+The default configuration for ``feeds`` field is the following:
+```javascript
+  "crimeservers" => {
+      "active" => false,
+      "feed_type" => "test",
+      "interval" => 900,
+    },
+    "botips" => {
+      "active" => false,
+      "feed_type" => "test",
+      "interval" => 600
+    }
+```
+
+
+#### Example
+
+```javascript
+input {
+     blueliv { api_key => "7da63a74-bece-42b0-843b-15728e44740d"
+               feeds => { "botips" => {
+                                "active" => true
+                                "feed_type" => "non_pos"
+                          }
+                          "crimeservers" => {
+                                "active" => true
+                                "feed_type" => "all"
+                          }
+              }
+            }
+}
+```
+Be aware that if you do not specify a given field, the default value will be configured. In this case, we did not touch the ``interval`` field for the feeds, so the defaults will apply.
+
+## Need Help?
+
+Need help? Send us an email to community@blueliv.com
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/inputs/blueliv.rb

@@ -0,0 +1,208 @@
+#!/usr/bin/env ruby
+# coding: UTF-8
+
+require "date"
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "rest-client"
+require "securerandom"
+
+ONE_DAY_IN_SECONDS = 86400
+
+RESOURCES = {
+  :crimeservers => {
+    :items => "crimeServers",
+    :endpoint => "/v1/crimeserver",
+    :feeds => {
+      :all => {
+        900 => "/last",
+        3600 => "/recent",
+        ONE_DAY_IN_SECONDS => "/online"
+      },
+      :test => {
+        900 => "/test"
+      }
+    }
+  },
+  :botips => {
+    :items => "ips",
+    :endpoint => "/v1/ip",
+    :feeds => {
+      :non_pos => {
+        600 => "/recent",
+        3600 => "/last"
+      },
+      :pos => {
+        600 => "/pos/recent",
+        3600 => "/pos/last"
+      },
+      :full => {
+        600 => "/full/recent",
+        3600 => "/full/last"
+      },
+      :test => {
+        600 => "/test"
+      }
+    }
+  }
+}
+
+DEFAULT_CONFIG = {
+  "crimeservers" => {
+      "active" => false,
+      "feed_type" => "test",
+      "interval" => 900,
+      "initialize" => true
+    },
+    "botips" => {
+      "active" => false,
+      "feed_type" => "test",
+      "interval" => 600
+    }
+}
+
+INITIALIZE_FILE = "blueliv.ini"
+FAILURE_SLEEP = 5 # seconds
+
+class LogStash::Inputs::Blueliv < LogStash::Inputs::Base
+  attr_accessor :auth, :timeout
+
+  config_name "blueliv"
+
+  @contact = "community@blueliv.com"
+
+  default :codec, "plain"
+
+  config :api_key, :validate => :string, :default => ""
+  config :api_url, :validate => :string, :default => "https://api.blueliv.com"
+  config :http_timeout, :validate => :number, :default => 500
+
+  config :feeds, :validate => :hash, :default => {}
+
+  public
+  def register
+    merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
+
+    @auth = "Bearer #{api_key}"
+    @feeds = DEFAULT_CONFIG.merge(feeds, &merger)
+    @timeout = http_timeout
+  end
+
+  def run(queue)
+      threads = []
+      @feeds.each do |name, conf|
+        if feeds[name]["active"]
+          if feeds[name]["initialize"]
+            if not db_updated?(DateTime.now)
+              threads << Thread.new do
+                url, interval = get_url(name, @feeds[name]["feed_type"], ONE_DAY_IN_SECONDS)
+                get_feed(queue, name, url) { write_last_update_db(DateTime.now) }
+                url, interval = get_url(name, @feeds[name]["feed_type"], @feeds[name]["interval"])
+                get_feed_each(queue, name, url, interval) { write_last_update_db(DateTime.now) }
+              end
+            end
+          else
+            url, interval = get_url(name, @feeds[name]["feed_type"], @feeds[name]["interval"])
+            threads << Thread.new(get_feed_each(queue, name, url, interval))
+          end
+        end
+      end
+
+      threads.map {|t| t.join}
+  end
+
+  def db_updated?(now)
+    result = false
+    if File.exists?(INITIALIZE_FILE)
+      File.open(INITIALIZE_FILE, "r") do |file|
+        file.each do |line|
+          unless line.strip.start_with? "#"
+            begin
+              time = DateTime.strptime(line.strip, "%s")
+              result = ((now - time) * ONE_DAY_IN_SECONDS).to_i < ONE_DAY_IN_SECONDS
+              break
+            rescue Exception => e
+              @logger.error(e)
+            end
+          end
+        end
+      end
+    end
+
+    result
+  end
+
+  def write_last_update_db(date)
+    File.open(INITIALIZE_FILE, "w") do |file|
+      file.write("# GENERATED FILE. DO NOT EDIT IT\n")
+      file.write("#{date.to_time.to_i}\n")
+    end
+  end
+
+  def get_url(name, feed_type, feed_interval)
+    base_url = @api_url + RESOURCES[name.to_sym][:endpoint]
+
+    if RESOURCES[name.to_sym][:feeds].key?(feed_type.to_sym)
+      feed_lookup = RESOURCES[name.to_sym][:feeds][feed_type.to_sym]
+    else
+      raise ArgumentError, "Feed #{feed_type} does not exist!"
+    end
+
+    interval = nil
+    feed_lookup.keys.each do |k|
+      if feed_interval <= k
+        interval = k
+        break
+      end
+    end
+    interval = feed_lookup.keys.max if interval == nil
+
+    return base_url + feed_lookup[interval], interval
+  end
+
+  def get_feed_each(queue, name, url, interval, &block)
+    loop do
+      get_feed(queue, name, url, &block)
+      sleep(interval)
+    end
+  end
+
+  def get_feed(queue, name, url, &block)
+    loop do
+      begin
+        response = client.get url, :Authorization => @auth, :timeout => @timeout
+        response_json = JSON.parse(response.body)
+        items = response_json[RESOURCES[name.to_sym][:items]]
+        items.each do |it|
+          it["location"] = [it["longitude"].to_f, it["latitude"].to_f]
+          collection = RESOURCES[name.to_sym][:items].downcase
+          it["@collection"] = collection
+          it["_id"] = SecureRandom.base64(32) unless it.has_key?("_id")
+          evt =  LogStash::Event.new(it)
+          decorate(evt)
+          queue << evt
+        end
+        block.call if block
+        break
+      rescue RestClient::Exception => e
+        @logger.error(e)
+        case e.http_code
+          when 401, 403
+            @logger.info("You do not have access to this resource! Please contact #{@contact}")
+            break
+          else
+            @logger.info("Will retry in #{FAILURE_SLEEP} seconds")
+            sleep(FAILURE_SLEEP)
+        end
+      rescue Exception => e
+        @logger.info("Will retry in #{FAILURE_SLEEP} seconds")
+        sleep(FAILURE_SLEEP)
+      end
+    end
+  end
+
+  def client
+    RestClient
+  end
+
+end

data/logstash-input-blueliv.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = "logstash-input-blueliv"
+  s.version = "0.1.0"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program"
+  s.authors = ["Blueliv"]
+  s.email = 'community@blueliv.com'
+  s.homepage = "http://github.com/Blueliv/logstash-input-blueliv"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_runtime_dependency "rest-client", "~> 1.8.0"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/inputs/blueliv_spec.rb

@@ -0,0 +1,82 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/blueliv"
+require "json"
+
+describe LogStash::Inputs::Blueliv do
+  let(:settings) {
+    {
+      "api_key" => "mykey",
+      "feeds" => {
+          "crimeservers" => {
+            "active" => false,
+            "feed_type" => "test",
+            "interval" => 900,
+            "initialize" => true
+          },
+          "botips" => {
+            "active" => false,
+            "feed_type" => "test",
+            "interval" => 600
+          }
+      }
+    }
+  }
+
+  let(:authorization) {
+    "Bearer mykey"
+  }
+
+  let(:timeout) {
+    500
+  }
+
+  let(:url) {
+    "https://api.blueliv.com/v1/ip/full/last"
+  }
+
+  let(:feed_name) {
+    "botips"
+  }
+
+  let(:mock_response) {
+    RestClient::Response.create(
+      {
+        "ips" => [
+            {
+              "field1" => "this",
+              "longitude" => 0.0,
+              "latitude" => 0.0
+            }
+          ],
+        "meta" => {
+          "totalSize" =>  1,
+          "updated" => "2015-07-01T10:40:00+0000",
+          "nextUpdate" => "2015-07-01T10:50:00+0000"
+        }
+      }.to_json, nil, nil, nil)
+  }
+
+  context "with json codec, when server responds" do
+    it "should not add an event if user is not authorized" do
+       subject = LogStash::Inputs::Blueliv.new()
+       logstash_queue = Queue.new
+
+       subject.get_feed logstash_queue, feed_name, url
+       expect(logstash_queue.size).to eq(0)
+       subject.teardown
+    end
+
+    it "should add an event if user is authorized" do
+       subject = LogStash::Inputs::Blueliv.new()
+       logstash_queue = Queue.new
+       subject.auth = authorization
+       subject.timeout = timeout
+
+       allow(subject.client).to receive(:get).with(url, :Authorization => authorization, :timeout => timeout).and_return(mock_response)
+       subject.get_feed logstash_queue, feed_name, url
+       expect(logstash_queue.size).to eq(1)
+       subject.teardown
+    end
+  end
+end
+

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-blueliv
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Blueliv
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+  name: rest-client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-blueliv. This gem is not a stand-alone program
+email: community@blueliv.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/inputs/blueliv.rb
+- logstash-input-blueliv.gemspec
+- spec/inputs/blueliv_spec.rb
+homepage: http://github.com/Blueliv/logstash-input-blueliv
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: This plugin allows users to access Blueliv Crime Servers and Bot IPs feeds.
+test_files:
+- spec/inputs/blueliv_spec.rb