logstash-output-monasca_log_api 0.3.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8a31e2b41940d7b7dec5eb88250d4bc27ce01e54
+  data.tar.gz: 71eecb218d00a0b32f762202d34d52f7bfc93a89
+SHA512:
+  metadata.gz: 95ec1d08625205ed811036c5b7eb142d3970901a693dc5c8455a1e41a7a1c799e4eefdb602a91e73fdbedf1864514229a040361156431f01d088254f5eaa07ce
+  data.tar.gz: ab6ab077a31d5587cf8eae2063f1a05a6d7349d3ed179a912879119407947c6c94cba9827ba38a09f189f78065f043d60750ab48dc4a11d925fb45d801f2d28d

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,11 @@
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.

data/README.md

@@ -0,0 +1,167 @@
+# Logstash Output Monasca-Log-Api Plugin
+
+## Build from source
+
+### Requirements
+
+* JRuby (I recommend to use Ruby Version Manager (RVM) How to install? ->  https://rvm.io/rvm/install)
+* JDK
+* Git
+* bundler
+
+### How to install the requirements
+
+#### Ubuntu 14.04
+
+##### RVM
+```bash
+gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+\curl -sSL https://get.rvm.io | bash
+source /home/vagrant/.rvm/scripts/rvm
+```
+
+#### Git
+```bash
+sudo apt-get install git
+```
+
+#### JDK
+```bash
+sudo apt-get install default-jdk
+```
+
+#### JRuby
+```bash
+rvm install jruby
+```
+
+### Clone project
+
+```bash
+git clone https://github.com/FujitsuEnablingSoftwareTechnologyGmbH/logstash-output-monasca_api.git
+```
+
+### Use rvm jruby
+```bash
+rvm use jruby
+```
+
+### Install bundler
+```bash
+gem install bundler
+```
+
+### Fetch dependencies
+```bash
+bundle install
+```
+
+## Unit tests
+
+### Run unit tests with:
+```bash
+bundle exec rspec
+```
+
+### Run specified test
+```bash
+bundle exec rspec spec/outputs/monasca/monasca_log_api_client_spec.rb
+```
+
+### Run coverage
+```bash
+JRUBY_OPTS="-Xcli.debug=true --debug" bundle exec rspec
+```
+
+Coverage report can be found in ./coverage
+
+## Deploy Plugin to logstash
+
+### Build Gemfile
+First we need to create a Gemfile.
+
+What is a Gemfile? Gems are packages for the Ruby programming language. A Gemfile defines all dependencies which are neccessary to build the product.
+
+How to build a Gemfile? Run this:
+```bash
+gem build logstash-output-monasca_log_api.gemspec
+```
+
+### Deploy Gemfile to logstash
+
+* [Download logstash](http://download.elastic.co/logstash/logstash/logstash-1.5.0.tar.gz) (>=1.5.0)
+* Extract logstash and navigate into the folder
+* Add this line to the Gemfile
+
+  ```bash
+  gem "logstash-output-monasca_log_api", :path => "/vagrant_home/cloudmonitor/logstash-output-monasca_log_api"
+  ```
+
+  * __logstash-output-monasca_log_api__ = name of the gem
+  * __/vagrant_home/cloudmonitor/logstash-output-monasca_log_api__ = Path to git workspace
+
+Run this command from logstash folder to install the plugin
+```bash
+bin/plugin install --no-verify
+```
+
+Verify installed plugins:
+With ``bin/plugin list`` you can check installed plugins. There should be ``logstash-output-monasca_log_api``.
+
+## Start logstash output plugin
+
+### Configuration
+
+Save the configfile wherever you like. For example ~/logstash.conf
+
+| name | description | required | example |
+| - | - | - | - |
+| monasca_log_api | monasca log api url | yes | http://192.168.10.4:8080 |
+| keystone_api | keystone api url | yes | http://192.168.10.5:5000 |
+| project_name | User-credentials: keystone project name | yes | mini-mon |
+| username | User-credentials: keystone username | yes | admin-agent |
+| password | User-credentials: keystone user password | yes | password |
+| domain_id | User-credentials: keystone user domain-id | yes | default |
+| dimensions | Dictionary of key-value pairs to describe logs | no | hostname: monasca, ip: 192.168.10.4 |
+| application_type_key | Application name | no | monasca |
+
+#### Example file
+```bash
+input {
+  stdin { }
+}
+output {
+  monasca_log_api {
+    monasca_log_api => "http://192.168.10.4:8080"
+    keystone_api => "http://192.168.10.5:5000"
+    project_name => "mini-mon"
+    username => "admin-agent"
+    password => "password"
+    domain_id => "default"
+    dimensions => "hostname: elkstack, ip: 192.168.10.4"
+    application_type_key => "monasca"
+  }
+}
+```
+
+Run
+```bash
+bin/logstash -f ~/logstash.conf
+```
+
+Run in debug mode
+```bash
+bin/logstash -f ~/logstash.conf --debug
+```
+
+Specify log output file
+```bash
+bin/logstash -f ~/logstash.conf -l /var/log/monasca/log/agent/test-log-agent.log
+```
+### Logstash Input plugins
+https://www.elastic.co/guide/en/logstash/current/input-plugins.html
+
+## Open tasks
+* Language translations (Replace hardcoded String messages with a configuration/language file)
+* Exception handling (monasca-api requests)
+* Contribute to logstash http://www.elastic.co/guide/en/logstash/master/_how_to_write_a_logstash_output_plugin.html#_contributing_your_source_code_to_ulink_url_https_github_com_logstash_plugins_logstash_plugins_ulink_4

data/lib/logstash/outputs/helper/url_helper.rb

@@ -0,0 +1,28 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require 'uri'
+
+module LogStash::Outputs
+  module Helper
+    class UrlHelper
+      def self.generate_url(host, path)
+        uri = URI.parse(host)
+        URI::HTTP.new(uri.scheme, nil, uri.host, uri.port, nil, path, nil, nil, nil)
+      end
+    end
+  end
+end

data/lib/logstash/outputs/keystone/keystone_client.rb

@@ -0,0 +1,69 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require 'logstash/environment'
+require 'logger'
+
+require_relative '../helper/url_helper'
+require_relative './token'
+
+# This class connects to keystone and authenticates an user.
+# If successful authenticated, the class returns a token object
+module LogStash::Outputs
+  module Keystone
+    class KeystoneClient
+
+      @client = nil
+
+      def initialize(host)
+        @logger = Cabin::Channel.get(LogStash)
+        @client = get_client(host, '/v3/auth/tokens')
+      end
+
+      def authenticate(domain_id, username, password, project_name)
+        auth_hash = generate_hash(domain_id, username, password, project_name)
+        response = request(auth_hash)
+        handle_response(response)
+      end
+
+      private
+
+      def request(auth_hash)
+        @client.post(auth_hash, :content_type => 'application/json', :accept => 'application/json')
+      end
+
+      def get_client(host, path)
+        RestClient::Resource.new(LogStash::Outputs::Helper::UrlHelper.generate_url(host, path).to_s)
+      end
+
+      def generate_hash(domain_id, username, password, project_name)
+        "{\"auth\":{\"identity\":{\"methods\":[\"password\"],\"password\":{\"user\":{\"domain\":{\"id\":\"#{domain_id}\"},\"name\":\"#{username}\",\"password\":\"#{password}\"}}},\"scope\":{\"project\":{\"domain\":{\"id\":\"#{domain_id}\"},\"name\":\"#{project_name}\"}}}}"
+      end
+
+      def handle_response(response)     
+        case response.code
+        when 201
+          expires_at = DateTime.parse(JSON.parse(response.body)["token"]["expires_at"])
+          @logger.debug("Authentication succeed: code=#{response.code}, auth-token=#{response.headers[:x_subject_token]}, expires_at=#{expires_at.to_time}")
+          Token.new(response.headers[:x_subject_token], expires_at)    
+        else
+          @logger.info("Authentication failed. Response=#{response}")
+        end
+      end
+
+    end
+  end
+end

data/lib/logstash/outputs/keystone/token.rb

@@ -0,0 +1,31 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+module LogStash::Outputs
+  module Keystone
+    class Token
+      attr_accessor :id, :expire_at
+      def initialize id, expire_at
+        @id = id
+        @expire_at = expire_at
+      end
+
+      def ==(another_token)
+        self.id == another_token.id && self.expire_at == another_token.expire_at
+      end
+    end
+  end
+end

data/lib/logstash/outputs/monasca/monasca_log_api_client.rb

@@ -0,0 +1,78 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require 'rest-client'
+require 'logger'
+
+# relative requirements
+require_relative '../helper/url_helper'
+
+# This class creates a connection to monasca-api
+module LogStash::Outputs
+  module Monasca
+    class MonascaLogApiClient
+
+      SUPPORTED_API_VERSION = %w(2.0)
+
+      def initialize(host, version)
+        @logger = Cabin::Channel.get(LogStash)
+        @rest_client_url = LogStash::Outputs::Helper::UrlHelper.generate_url(host, '/' + check_version(version)).to_s
+        @rest_client = RestClient::Resource.new(@rest_client_url)
+      end
+
+      # Send log events to monasca-api, requires token
+      def send_event(event, data, token, dimensions, application_type=nil)
+        begin
+          request(event, data, token, dimensions, application_type)
+          @logger.debug("Successfully send event=#{event}, with token=#{token} and dimensions=#{dimensions} to monasca-api")
+        rescue => e
+          @logger.warn('Sending event to monasca-log-api threw exception', :exceptionew => e)
+        end
+      end
+
+      private
+
+      def request(event, data, token, dimensions, application_type)
+        post_headers = {
+            :x_auth_token => token,
+            :content_type => 'application/json',
+        }
+        if dimensions
+          post_headers[:x_dimensions] = dimensions
+        end
+
+        if application_type
+          post_headers[:x_application_type] = application_type
+        end
+
+        @logger.debug('Sending data to ', :url => @rest_client_url)
+        @rest_client['log']['single'].post(data, post_headers)
+      end
+
+
+      def check_version(version)
+        tmp_version = version.sub('v','')
+
+        unless SUPPORTED_API_VERSION.include? tmp_version
+          raise "#{tmp_version} is not supported, supported versions are #{SUPPORTED_API_VERSION}"
+        end
+
+        version
+      end
+
+    end
+  end
+end

data/lib/logstash/outputs/monasca_log_api.rb

@@ -0,0 +1,134 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require 'logstash/outputs/base'
+require 'logstash/namespace'
+require 'vine'
+
+# relative requirements
+require_relative 'monasca/monasca_log_api_client'
+require_relative 'keystone/keystone_client'
+
+# This Logstash Output plugin, sends events to monasca-api.
+# It authenticates against keystone and gets a token.
+# The token is used to authenticate against the monasca-api and send log events.
+class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
+  config_name 'monasca_log_api'
+
+  # monasca-api host and port configuration
+  config :monasca_log_api, :validate => :string, :required => true
+  config :monasca_log_api_version, :validate => :string, :required => false, :default => "v2.0"
+
+  # keystone host and port configuration
+  config :keystone_api, :validate => :string, :required => true
+  # keystone user configuration
+  config :project_name, :validate => :string, :required => true
+  config :username, :validate => :string, :required => true
+  config :password, :validate => :string, :required => true
+  config :domain_id, :validate => :string, :required => true
+
+  config :dimensions, :validate => :string, :default => nil
+  config :application_type_key, :validate => :string, :default => 'type'
+
+  attr_accessor :token
+
+  default :codec, 'json'
+
+  public
+  def register
+    @keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new keystone_api
+    @monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new monasca_log_api, monasca_log_api_version
+    @token = get_token
+
+    @logger.info('Registering keystone user', :username => @username, :project_name => @project_name)
+
+    @codec.on_event do |event, data|
+      check_token
+      send_event(event, data, dimensions)
+    end
+
+  end # def register
+
+  def receive(event)
+    return unless output?(event)
+    if event == LogStash::SHUTDOWN
+      finished
+      return
+    end
+    @codec.encode(event)
+  end # def receive
+
+  def finished
+    if @shutdown_queue
+      @logger.info('Sending shutdown event to agent queue', :plugin => self.class.config_name)
+      @shutdown_queue << self
+    end
+    if @plugin_state != :finished
+      @logger.info('Plugin is finished', :plugin => self.class.config_name)
+      @plugin_state = :finished
+    end
+  end
+
+  def shutdown(queue)
+    teardown
+    @logger.info('Received shutdown signal', :plugin => self.class.config_name)
+
+    @shutdown_queue = queue
+    if @plugin_state == :finished
+      finished
+    else
+      @plugin_state = :terminating
+    end
+  end
+
+  private
+  def check_token
+    now = DateTime.now + Rational(1, 1440)
+    if now >= @token.expire_at
+      @token = get_token
+      @logger.info('Token expired. New token requested')
+    end
+  end
+
+  def send_event(event, data, dimensions)
+    @monasca_log_api_client.send_event(event, data, @token.id, dimensions, get_application_type(event)) if event and @token.id and data
+  end
+
+  def get_application_type(event)
+
+    # support for nested.keys.with.lists.0
+    if @application_type_key.include? '.'
+      split_key = @application_type_key.split('.')
+      event_value = event[split_key[0]]
+      access_key = split_key.slice(1..-1).join('.')
+
+      app_type = event_value.access(access_key)
+
+    else
+      app_type = event[@application_type_key]
+    end
+
+    @logger.debug(
+        'Retrieved application_type', :application_type => app_type, :application_type_key => @application_type_key
+    )
+    app_type
+  end
+
+  def get_token
+    @keystone_client.authenticate(domain_id, username, password, project_name)
+  end
+
+end # class LogStash::Outputs::Example

data/logstash-output-monasca_log_api.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-monasca_log_api'
+  s.version = '0.3.3'
+  s.licenses = ['Apache License 2.0']
+  s.summary = 'This gem is a logstash output plugin to connect via http to monasca-log-api.'
+  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
+  s.authors = ['Fujitsu Enabling Software Technology GmbH']
+  s.email = 'kamil.choroba@est.fujitsu.com,tomasz.trebski@ts.fujitsu.com'
+  s.require_paths = ['lib']
+  s.homepage = 'https://github.com/FujitsuEnablingSoftwareTechnologyGmbH/logstash-output-monasca_api'
+
+  # Files
+  s.files = Dir["lib/**/*", "spec/**/*", "*.gemspec", "*.md", "Gemfile", "LICENSE"]
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '~> 1.5'
+  s.add_runtime_dependency 'logstash-codec-plain', '~> 0.1.6'
+  s.add_runtime_dependency 'logstash-codec-json', '~> 0.1.6'
+  s.add_runtime_dependency 'rest-client', '~> 1.8'
+  s.add_runtime_dependency 'vine', '~> 0.2'
+  s.add_development_dependency 'logstash-devutils', '~> 0.0.14'
+  s.add_development_dependency 'simplecov', '~> 0.10'
+end

data/spec/outputs/helper/url_helper_spec.rb

@@ -0,0 +1,34 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require_relative '../spec_helper'
+
+describe LogStash::Outputs::Helper::UrlHelper do
+
+  describe ".generate_url" do
+    it "generates a URI::HTTP object" do
+      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://192.168.10.5:8080', '/v2.0')).to be_a URI::HTTP
+    end
+
+    it "should match a http url" do
+      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://192.168.10.5:8080', '/v2.0').to_s).to eq('http://192.168.10.5:8080/v2.0')
+      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://est.fujitsu:40', nil).to_s).to eq('http://est.fujitsu:40')
+      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://est.fujitsu:80', nil).to_s).to eq('http://est.fujitsu')
+      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('https://192.168.10.5:8080', '/v2.0').to_s).to eq('https://192.168.10.5:8080/v2.0')
+    end
+  end
+
+end

data/spec/outputs/keystone/keystone_client_spec.rb

@@ -0,0 +1,81 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require_relative '../spec_helper'
+
+describe LogStash::Outputs::Keystone::KeystoneClient do
+
+  let (:auth_hash) { "{\"auth\":{\"identity\":{\"methods\":[\"password\"],\"password\":{\"user\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi-operator\",\"password\":\"password\"}}},\"scope\":{\"project\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi\"}}}}" }
+
+  let (:ok_response) { stub_response(201, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"token\":{\"methods\":[\"password\"],\"roles\":[{\"id\":\"9fe2ff9ee4384b1894a90878d3e92bab\",\"name\":\"_member_\"},{\"id\":\"4e9ef1ffe73446c6b02f8fce0585c307\",\"name\":\"monasca-user\"}],\"expires_at\":\"2015-05-26T08:55:36.774122Z\",\"project\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"1051bd27b9394120b26d8b08847325c0\",\"name\":\"mini-mon\"},\"user\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"06ecc9869b8e4846b2fce3e5759ba4af\",\"name\":\"mini-mon\"},\"audit_ids\":[\"ORap7R56S2S-p6tFVeMkpg\"],\"issued_at\":\"2015-05-26T07:55:36.774146Z\"}}") }
+
+  let (:failed_response) { stub_response(401, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"error\": {\"message\": \"Could not find project: f8cdafb7dce94444ad781a53ddaff693 (Disable debug mode to suppress these details.)\", \"code\": 401, \"title\": \"Unauthorized\"}}") }
+
+  context 'when initializing' do
+    it 'then it should register without exceptions' do
+      expect {LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')}.to_not raise_error
+    end
+
+    it "returns a failure if arguments are missing" do
+      expect {LogStash::Outputs::Keystone::KeystoneClient.new}.to raise_exception(ArgumentError)
+    end
+  end
+
+  context 'when authenticates' do
+    it 'then it should request to keystone' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .with(auth_hash, :content_type => 'application/json', :accept => 'application/json')
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
+      allow(keystone_client).to receive(:handle_response).and_return(LogStash::Outputs::Keystone::Token.new('abc', DateTime.now + Rational(5, 1440)))
+      keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+    end
+
+    it 'then it should create a new token' do
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
+      allow(keystone_client).to receive(:request).and_return(ok_response)
+      expected = LogStash::Outputs::Keystone::Token.new('f8cdafb7dce94444ad781a53ddaff693', DateTime.parse("2015-05-26T08:55:36.774122Z"))
+      actual = keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+      expect(actual).to eq(expected)
+    end
+  end
+
+  context 'when authentication failed' do
+    it 'then it should not create a new token' do
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
+      allow(keystone_client).to receive(:request).and_return(failed_response)
+      actual = keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+      expect(actual).to eq(nil)
+    end
+
+    it 'then it should log an information' do
+      expect_any_instance_of(Cabin::Channel).to receive(:info)
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
+      allow(keystone_client).to receive(:request).and_return(failed_response)
+      keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+    end
+  end
+
+  private
+
+  def stub_response(code, headers, body)
+    response = double
+    allow(response).to receive(:code) { code }
+    allow(response).to receive(:headers) { headers }
+    allow(response).to receive(:body) { body }
+    response
+  end
+end

data/spec/outputs/keystone/token_spec.rb

@@ -0,0 +1,38 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require_relative '../spec_helper'
+
+describe LogStash::Outputs::Keystone::Token do
+
+  describe "#new" do
+    it "takes two parameters and returns a Token object" do
+      token = LogStash::Outputs::Keystone::Token.new('token-id', DateTime.now)
+      expect(token).to be_a(LogStash::Outputs::Keystone::Token)
+    end    
+  end
+
+  describe "#==" do
+    it "should equal to another token instance" do
+      token = LogStash::Outputs::Keystone::Token.new('token-id', DateTime.parse("2015-05-26T08:55:36.774122Z"))
+      second_token = LogStash::Outputs::Keystone::Token.new('token-id', DateTime.parse("2015-05-26T08:55:36.774122Z"))
+      third_token = LogStash::Outputs::Keystone::Token.new('token-id', DateTime.parse("2015-05-30T08:55:36.774122Z"))
+
+      expect(token).to eq(second_token)
+      expect(third_token).to_not eq(second_token)
+    end
+  end
+end

data/spec/outputs/monasca/monasca_api_client_spec.rb

@@ -0,0 +1,105 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require_relative '../spec_helper'
+
+describe LogStash::Outputs::Monasca::MonascaLogApiClient do
+  let (:version) { "v2.0" }
+
+  let (:auth_hash) { "{\"auth\":{\"identity\":{\"methods\":[\"password\"],\"password\":{\"user\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi-operator\",\"password\":\"password\"}}},\"scope\":{\"project\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi\"}}}}" }
+
+  let (:ok_response) { stub_response(201, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"token\":{\"methods\":[\"password\"],\"roles\":[{\"id\":\"9fe2ff9ee4384b1894a90878d3e92bab\",\"name\":\"_member_\"},{\"id\":\"4e9ef1ffe73446c6b02f8fce0585c307\",\"name\":\"monasca-user\"}],\"expires_at\":\"2015-05-26T08:55:36.774122Z\",\"project\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"1051bd27b9394120b26d8b08847325c0\",\"name\":\"mini-mon\"},\"user\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"06ecc9869b8e4846b2fce3e5759ba4af\",\"name\":\"mini-mon\"},\"audit_ids\":[\"ORap7R56S2S-p6tFVeMkpg\"],\"issued_at\":\"2015-05-26T07:55:36.774146Z\"}}") }
+
+  let (:failed_response) { stub_response(401, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"error\": {\"message\": \"Could not find project: f8cdafb7dce94444ad781a53ddaff693 (Disable debug mode to suppress these details.)\", \"code\": 401, \"title\": \"Unauthorized\"}}") }
+
+  let (:data) { LogStash::Event.new({'message' => '2015-08-13 08:36:59,316 INFO monasca_notification.main Received signal 17, beginning graceful shutdown.',
+                                      '@version' => '1',
+                                      '@timestamp' => '2015-08-13T08:37:00.287Z',
+                                      'type' => 'notification',
+                                      'service' => 'notification',
+                                      'host' => 'monasca',
+                                      'path' => '/var/log/monasca/notification/notification.log'}) }
+
+  let (:token) { LogStash::Outputs::Keystone::Token.new('abc', DateTime.now + Rational(5, 1440)) }
+
+  let (:dimensions) { { 'hostname' => 'monasca' } }
+
+  context 'when initializing' do
+    it 'then it should register without exceptions' do
+      expect {LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', version)}.to_not raise_error
+    end
+
+    it "returns a failure if arguments are missing" do
+      # noinspection RubyArgCount
+      expect {LogStash::Outputs::Monasca::MonascaLogApiClient.new}.to raise_exception(ArgumentError)
+    end
+  end
+
+  context 'when sending events' do
+  	it 'with dimensions then it should request monasca' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .with(data, :x_auth_token => token, :content_type => 'application/json', :x_dimensions => dimensions)
+      monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', version)
+      monasca_log_api_client.send_event(nil, data, token, dimensions)
+  	end
+
+  	it 'without dimensions then it should request monasca' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .with(data, :x_auth_token => token, :content_type => 'application/json')
+      monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', version)
+      monasca_log_api_client.send_event(nil, data, token, nil)
+    end
+
+    it 'with application_type then it should request monasca with that value' do
+      app_type = 'someapp'
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .with(data, :x_auth_token => token, :content_type => 'application/json', :x_application_type => app_type)
+      monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', version)
+      monasca_log_api_client.send_event(nil, data, token, nil, app_type)
+    end
+  end
+
+  context 'when sending events failes' do
+  	it 'then it should be rescued and a warn log printed' do
+  	  expect_any_instance_of(Cabin::Channel).to receive(:warn)
+      monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', version)
+      allow(monasca_log_api_client).to receive(:request).and_raise('an_error')
+      monasca_log_api_client.send_event(nil, data, token, dimensions)
+  	end
+  end
+
+  context 'api version checking' do
+
+    let (:supported_version) { %w(v2.0) }
+
+    it 'should pass for correct version' do
+      supported_version.each { |ver|
+        expect {LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', ver)}.to_not raise_error
+      }
+    end
+
+    it 'should pass if version does not specify v' do
+      ver = '2.0'
+      expect {LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', ver)}.to_not raise_error
+    end
+
+    it 'should fail for unsupported version' do
+      ver = 'v4.0'
+      expect {LogStash::Outputs::Monasca::MonascaLogApiClient.new('hostname:8080', ver)}.to raise_exception
+    end
+  end
+
+end

data/spec/outputs/monasca_api_spec.rb

@@ -0,0 +1,131 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe 'outputs/monasca_log_api' do
+
+  let (:expected_application_type) { 'notification' }
+
+  let (:event) { LogStash::Event.new({'message' => '2015-08-13 08:36:59,316 INFO monasca_notification.main Received signal 17, beginning graceful shutdown.',
+                                      '@version' => '1',
+                                      '@timestamp' => '2015-08-13T08:37:00.287Z',
+                                      'type' => expected_application_type,
+                                      'service' => 'notification',
+                                      'host' => 'monasca',
+                                      'path' => '/var/log/monasca/notification/notification.log'}) }
+
+  let (:shutdown_event) { LogStash::SHUTDOWN }
+
+  let (:simple_config) { {'monasca_log_api' => 'http://192.168.10.4:8080',
+      'keystone_api' => 'http://192.168.10.5:5000',
+      'domain_id' => 'abadcf984cf7401e88579d393317b0d9',
+      'username' => 'username',
+      'password' => 'password',
+      'project_name' => 'project_name'} }
+
+  let (:token_id) { LogStash::Outputs::Keystone::Token.new('abc', DateTime.now + Rational(5, 1440)) }
+
+  let (:old_token_id) { LogStash::Outputs::Keystone::Token.new('def', DateTime.now - Rational(5, 1440)) }
+
+  context 'when initializing' do
+    it 'then it should register without exceptions' do
+      monasca_log_api = LogStash::Plugin.lookup('output', 'monasca_log_api').new(simple_config)
+      allow(monasca_log_api).to receive(:get_token).and_return(token_id)
+      expect {monasca_log_api.register}.to_not raise_error
+    end
+
+    it "should set a default application_type_key to \"type\"" do
+      monasca_log_api = LogStash::Plugin.lookup('output', 'monasca_log_api').new(simple_config)
+      expect(monasca_log_api.instance_variable_get('@application_type_key')).to eq("type")
+    end
+
+    it 'should use passed variable from config as application_type_key' do
+      expected_key = 'mountain_dew'
+      simple_config_copy = simple_config.clone
+      simple_config_copy['application_type_key'] = expected_key
+      monasca_log_api = LogStash::Plugin.lookup('output', 'monasca_log_api').new(simple_config_copy)
+
+      expect(monasca_log_api.instance_variable_get('@application_type_key')).to eq(expected_key)
+    end
+
+  end
+
+  context 'when receiving messages' do
+    it 'then it should be send to monasca-log-api' do
+      expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient).to receive(:send_event)
+        .with(event, event.to_hash.to_json, token_id.id, nil, expected_application_type)
+      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(simple_config)
+
+      allow(monasca_log_api).to receive(:get_token).and_return(token_id)
+      allow(monasca_log_api).to receive(:get_application_type).and_call_original
+
+      monasca_log_api.register
+      monasca_log_api.receive(event)
+    end
+
+    it 'then it should check the token_id and renew it when its deprecated' do
+      expect_any_instance_of(LogStash::Outputs::Keystone::KeystoneClient).to receive(:authenticate)
+        .with(simple_config['domain_id'], simple_config['username'], simple_config['password'], simple_config['project_name'])
+      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(simple_config)
+      allow(monasca_log_api).to receive(:get_token).and_return(old_token_id)
+      monasca_log_api.register
+      allow(monasca_log_api).to receive(:get_token).and_call_original
+      allow(monasca_log_api).to receive(:send_event)
+      monasca_log_api.receive(event)
+    end
+  end
+
+  context 'when given dictionary as type in received message' do
+    it 'should read application_type from it properly' do
+      type_dict = {
+          "application_type" => 'notification'
+      }
+      config = simple_config.merge({"application_type_key" => 'type.application_type'})
+
+      event = LogStash::Event.new({"message" => '2015-08-13 08:36:59,316 INFO monasca_notification.main Received signal 17, beginning graceful shutdown.',
+                                   "@version" => '1',
+                                   "@timestamp" => '2015-08-13T08:37:00.287Z',
+                                   "type" => type_dict,
+                                   "service" => 'notification',
+                                   "host" => 'monasca',
+                                   "path" => '/var/log/monasca/notification/notification.log',
+                                  })
+      expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient).to receive(:send_event)
+        .with(event, event.to_hash.to_json, token_id.id, nil, expected_application_type)
+      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(config)
+
+      allow(monasca_log_api).to receive(:get_token).and_return(token_id)
+      allow(monasca_log_api).to receive(:get_application_type).and_call_original
+
+      monasca_log_api.register
+      monasca_log_api.receive(event)
+    end
+
+  end
+
+  context 'when receiving SHUTDOWN message' do
+    it 'then it should be send to monasca-log-api' do
+      expect_any_instance_of(LogStash::Outputs::Monasca::MonascaLogApiClient).to_not receive(:send_event)
+        .with(shutdown_event, nil, token_id.id, nil)
+      monasca_log_api = LogStash::Outputs::MonascaLogApi.new(simple_config)
+      allow(monasca_log_api).to receive(:get_token).and_return(token_id)
+      monasca_log_api.register
+      monasca_log_api.receive(shutdown_event)
+    end
+  end
+
+end

data/spec/outputs/spec_helper.rb

@@ -0,0 +1,32 @@
+=begin
+Copyright 2015 FUJITSU LIMITED
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License
+is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+or implied. See the License for the specific language governing permissions and limitations under
+the License.
+=end
+
+# encoding: utf-8
+
+require 'simplecov'
+SimpleCov.start do
+  add_filter '/test/'
+  add_filter '/config/'
+  add_filter '/vendor/'
+  add_filter '/spec/'
+end
+
+require_relative '../../lib/logstash/outputs/monasca_log_api'
+require_relative '../../lib/logstash/outputs/keystone/keystone_client'
+require_relative '../../lib/logstash/outputs/keystone/token'
+require_relative '../../lib/logstash/outputs/monasca/monasca_log_api_client'
+require_relative '../../lib/logstash/outputs/helper/url_helper'
+
+require 'yaml'
+require 'rest-client'

metadata

@@ -0,0 +1,164 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-monasca_log_api
+version: !ruby/object:Gem::Version
+  version: 0.3.3
+platform: ruby
+authors:
+- Fujitsu Enabling Software Technology GmbH
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-12-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: vine
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.14
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.14
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: kamil.choroba@est.fujitsu.com,tomasz.trebski@ts.fujitsu.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/helper/url_helper.rb
+- lib/logstash/outputs/keystone/keystone_client.rb
+- lib/logstash/outputs/keystone/token.rb
+- lib/logstash/outputs/monasca/monasca_log_api_client.rb
+- lib/logstash/outputs/monasca_log_api.rb
+- logstash-output-monasca_log_api.gemspec
+- spec/outputs/helper/url_helper_spec.rb
+- spec/outputs/keystone/keystone_client_spec.rb
+- spec/outputs/keystone/token_spec.rb
+- spec/outputs/monasca/monasca_api_client_spec.rb
+- spec/outputs/monasca_api_spec.rb
+- spec/outputs/spec_helper.rb
+homepage: https://github.com/FujitsuEnablingSoftwareTechnologyGmbH/logstash-output-monasca_api
+licenses:
+- Apache License 2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: This gem is a logstash output plugin to connect via http to monasca-log-api.
+test_files:
+- spec/outputs/helper/url_helper_spec.rb
+- spec/outputs/keystone/keystone_client_spec.rb
+- spec/outputs/keystone/token_spec.rb
+- spec/outputs/monasca/monasca_api_client_spec.rb
+- spec/outputs/monasca_api_spec.rb
+- spec/outputs/spec_helper.rb