logstash-output-monasca_log_api 0.3.3 → 0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a31e2b41940d7b7dec5eb88250d4bc27ce01e54
-  data.tar.gz: 71eecb218d00a0b32f762202d34d52f7bfc93a89
+  metadata.gz: 59bde00a1b625fee42b79d0244c9f5bbb8852ca7
+  data.tar.gz: 906cf828c864a7b3f7d913d96cf26e47677afa10
 SHA512:
-  metadata.gz: 95ec1d08625205ed811036c5b7eb142d3970901a693dc5c8455a1e41a7a1c799e4eefdb602a91e73fdbedf1864514229a040361156431f01d088254f5eaa07ce
-  data.tar.gz: ab6ab077a31d5587cf8eae2063f1a05a6d7349d3ed179a912879119407947c6c94cba9827ba38a09f189f78065f043d60750ab48dc4a11d925fb45d801f2d28d
+  metadata.gz: 53bbbb42df6fcd83d4e7c31fbb93382607d497bcb69cd9585ae06578bf65542422ac661632061aedc71e86d9deeefa009811ea0048e11e944acf83a7e28ea751
+  data.tar.gz: 1154db9689c90dc9fadbe66076efe87c26f54ad04babcda105f6a18467b1056d10d05e16d59ae9f22acaa0092aa44a8e2a4d47bf9de273f07b34af1583bb803b

data/LICENSE

@@ -1,11 +1,13 @@
-Copyright 2015 FUJITSU LIMITED
+Copyright 2016 FUJITSU LIMITED
 
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
+Licensed under the Apache License, Version 2.0 (the "License"); you may not
+use this file except in compliance with the License. You may obtain a copy of
+the License at
 
 http://www.apache.org/licenses/LICENSE-2.0
 
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations under
+the License.

data/README.md

@@ -1,4 +1,12 @@
 # Logstash Output Monasca-Log-Api Plugin
+This module is a logstash-output-plugin for the Monasca Log Api.
+
+## Get latest stable version
+https://rubygems.org/gems/logstash-output-monasca_log_api
+
+```bash
+gem install logstash-output-monasca_log_api
+```
 
 ## Build from source
 
@@ -9,27 +17,15 @@
 * Git
 * bundler
 
-### How to install the requirements
-
-#### Ubuntu 14.04
+### Install requirements
 
-##### RVM
+#### RVM
 ```bash
 gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
 \curl -sSL https://get.rvm.io | bash
 source /home/vagrant/.rvm/scripts/rvm
 ```
 
-#### Git
-```bash
-sudo apt-get install git
-```
-
-#### JDK
-```bash
-sudo apt-get install default-jdk
-```
-
 #### JRuby
 ```bash
 rvm install jruby
@@ -41,7 +37,7 @@ rvm install jruby
 git clone https://github.com/FujitsuEnablingSoftwareTechnologyGmbH/logstash-output-monasca_api.git
 ```
 
-### Use rvm jruby
+### Use jruby
 ```bash
 rvm use jruby
 ```
@@ -75,8 +71,15 @@ JRUBY_OPTS="-Xcli.debug=true --debug" bundle exec rspec
 
 Coverage report can be found in ./coverage
 
+### Code style check
+```bash
+rubocop lib/
+```
+
 ## Deploy Plugin to logstash
 
+Source: [https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html#_building_and_testing_4](https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html#_building_and_testing_4)
+
 ### Build Gemfile
 First we need to create a Gemfile.
 
@@ -89,7 +92,7 @@ gem build logstash-output-monasca_log_api.gemspec
 
 ### Deploy Gemfile to logstash
 
-* [Download logstash](http://download.elastic.co/logstash/logstash/logstash-1.5.0.tar.gz) (>=1.5.0)
+* [Download logstash](https://download.elastic.co/logstash/logstash/logstash-2.2.2.tar.gz) (>=2.2.0)
 * Extract logstash and navigate into the folder
 * Add this line to the Gemfile
 
@@ -112,34 +115,57 @@ With ``bin/plugin list`` you can check installed plugins. There should be ``logs
 
 ### Configuration
 
-Save the configfile wherever you like. For example ~/logstash.conf
+Plugin name: monasca_log_api
 
-| name | description | required | example |
-| - | - | - | - |
-| monasca_log_api | monasca log api url | yes | http://192.168.10.4:8080 |
-| keystone_api | keystone api url | yes | http://192.168.10.5:5000 |
-| project_name | User-credentials: keystone project name | yes | mini-mon |
-| username | User-credentials: keystone username | yes | admin-agent |
-| password | User-credentials: keystone user password | yes | password |
-| domain_id | User-credentials: keystone user domain-id | yes | default |
-| dimensions | Dictionary of key-value pairs to describe logs | no | hostname: monasca, ip: 192.168.10.4 |
-| application_type_key | Application name | no | monasca |
+Save the configfile wherever you like. For example ~/logstash.conf
 
-#### Example file
+| name | description | type | required | default | example |
+| --- | --- | --- | --- | --- | --- |
+| monasca_log_api | monasca log api url | string | true | | http://192.168.10.4:8080 |
+| monasca_log_api_version | monasca log api version | string | false | v3.0 | |
+| keystone_api | keystone api url | string | true | | http://192.168.10.5:35357/v3 |
+| project_name | Keystone user credentials: project name | string | true | | monasca |
+| username | Keystone user credentials: username | string | true | | admin-agent |
+| password | Keystone user credentials: password | string | true | | password |
+| domain_id | Keystone user credentials: domain-id | string | true | | default |
+| dimensions | global array dimensions in form of key-value pairs to describe the monitored node | array | false | | ['app_type:kafka', 'priority:high'] |
+| num_of_logs | maximum number of logs that are send by one request to monasca-log-api | number | false | 125 | |
+| elapsed_time_sec | send logs if the maximum elapsed time in seconds is reached | number | false | 30 | |
+| delay | delay time in seconds to wait before checking the elapsed_time_sec again | number | false | 10 | |
+| max_data_size_kb | maximum size in kb of logs that are send by one request to monasca-log-api | number | false | 5120 | |
+
+#### Example configuration files
+
+##### Simple
 ```bash
-input {
-  stdin { }
+output {
+  monasca_log_api {
+    monasca_log_api => "http://192.168.10.4:8074"
+    keystone_api => "http://192.168.10.5:35357/v3"
+    project_name => "cmm"
+    username => "cmm-operator"
+    password => "admin"
+    domain_id => "default"
+  }
 }
+```
+
+##### Complete
+```bash
 output {
   monasca_log_api {
-    monasca_log_api => "http://192.168.10.4:8080"
-    keystone_api => "http://192.168.10.5:5000"
-    project_name => "mini-mon"
-    username => "admin-agent"
-    password => "password"
+    monasca_log_api => "http://192.168.10.4:8074"
+    monasca_log_api_version => "v3.0"
+    keystone_api => "http://192.168.10.5:35357/v3"
+    project_name => "cmm"
+    username => "cmm-operator"
+    password => "admin"
     domain_id => "default"
-    dimensions => "hostname: elkstack, ip: 192.168.10.4"
-    application_type_key => "monasca"
+    dimensions => ["hostname:kamil", "ip:10.10.10.10"]
+    num_of_logs => 100
+    delay => 1
+    elapsed_time_sec => 600
+    max_data_size_kb => 5120
   }
 }
 ```
@@ -158,10 +184,26 @@ Specify log output file
 ```bash
 bin/logstash -f ~/logstash.conf -l /var/log/monasca/log/agent/test-log-agent.log
 ```
-### Logstash Input plugins
-https://www.elastic.co/guide/en/logstash/current/input-plugins.html
+
+### Logstash File Input plugin
+https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html
+
+#### Configuration
+
+Local dimensions can be added with ```add_field``` setting
+
+```bash
+input {
+  file {
+    add_field => { "dimensions" => { "service" => "monasca-api" }}
+    add_field => { "dimensions" => { "language" => "java" }}
+    add_field => { "dimensions" => { "log_level" => "error" }}
+    path => "/var/log/monasca/api/error.log"
+    }
+  }
+```
 
 ## Open tasks
 * Language translations (Replace hardcoded String messages with a configuration/language file)
 * Exception handling (monasca-api requests)
-* Contribute to logstash http://www.elastic.co/guide/en/logstash/master/_how_to_write_a_logstash_output_plugin.html#_contributing_your_source_code_to_ulink_url_https_github_com_logstash_plugins_logstash_plugins_ulink_4
+* https support

data/lib/logstash/outputs/helper/url_helper.rb

@@ -1,16 +1,16 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
@@ -21,7 +21,8 @@ module LogStash::Outputs
     class UrlHelper
       def self.generate_url(host, path)
         uri = URI.parse(host)
-        URI::HTTP.new(uri.scheme, nil, uri.host, uri.port, nil, path, nil, nil, nil)
+        URI::HTTP.new(uri.scheme, nil, uri.host, uri.port, nil, path, nil,
+          nil, nil)
       end
     end
   end

data/lib/logstash/outputs/keystone/keystone_client.rb

@@ -1,16 +1,16 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
@@ -42,23 +42,52 @@ module LogStash::Outputs
       private
 
       def request(auth_hash)
-        @client.post(auth_hash, :content_type => 'application/json', :accept => 'application/json')
+        @client.post(
+          auth_hash,
+          :content_type => 'application/json',
+          :accept => 'application/json')
       end
 
       def get_client(host, path)
-        RestClient::Resource.new(LogStash::Outputs::Helper::UrlHelper.generate_url(host, path).to_s)
+        RestClient::Resource.new(
+          LogStash::Outputs::Helper::UrlHelper.generate_url(host, path).to_s)
       end
 
       def generate_hash(domain_id, username, password, project_name)
-        "{\"auth\":{\"identity\":{\"methods\":[\"password\"],\"password\":{\"user\":{\"domain\":{\"id\":\"#{domain_id}\"},\"name\":\"#{username}\",\"password\":\"#{password}\"}}},\"scope\":{\"project\":{\"domain\":{\"id\":\"#{domain_id}\"},\"name\":\"#{project_name}\"}}}}"
+        {
+          "auth"=>{
+            "identity"=>{
+              "methods"=>["password"],
+              "password"=>{
+                "user"=>{
+                  "domain"=>{"id"=>domain_id},
+                  "name"=>username,
+                  "password"=>password
+                }
+              }
+            },
+            "scope"=>{
+              "project"=>{
+                "domain"=>{"id"=>domain_id},
+                "name"=>project_name
+              }
+            }
+          }
+        }.to_json
       end
 
-      def handle_response(response)     
+      def handle_response(response)
         case response.code
         when 201
-          expires_at = DateTime.parse(JSON.parse(response.body)["token"]["expires_at"])
-          @logger.debug("Authentication succeed: code=#{response.code}, auth-token=#{response.headers[:x_subject_token]}, expires_at=#{expires_at.to_time}")
-          Token.new(response.headers[:x_subject_token], expires_at)    
+          expires_at = DateTime.parse(
+            JSON.parse(response.body)["token"]["expires_at"])
+
+          @logger.debug("Authentication succeed: code=#{response.code}, "\
+            "auth-token=#{response.headers[:x_subject_token]}, "\
+            "expires_at=#{expires_at.to_time}")
+
+          {:token => response.headers[:x_subject_token],
+            :expires_at => expires_at}
         else
           @logger.info("Authentication failed. Response=#{response}")
         end

data/lib/logstash/outputs/keystone/token.rb

@@ -1,31 +1,60 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
+require 'singleton'
+require 'logger'
+require_relative 'keystone_client'
+
 module LogStash::Outputs
   module Keystone
     class Token
-      attr_accessor :id, :expire_at
-      def initialize id, expire_at
+      include Singleton
+      attr_accessor :id, :expires_at, :keystone_client
+
+      def request_new_token(domain_id, username, password, project_name)
+        token = @keystone_client
+          .authenticate(domain_id, username, password, project_name)
+        set_token(token[:token], token[:expires_at])
+        @logger.info("New token requested")
+        @logger.debug("token=#{@id}, expire_at=#{@expires_at}")
+      end
+
+      def set_token id, expires_at
         @id = id
-        @expire_at = expire_at
+        @expires_at = expires_at
+      end
+
+      def set_keystone_client keystone_api
+        @keystone_client = LogStash::Outputs::Keystone::KeystoneClient
+          .new keystone_api
       end
 
-      def ==(another_token)
-        self.id == another_token.id && self.expire_at == another_token.expire_at
+      def initialize
+        @logger = Cabin::Channel.get(LogStash)
       end
+
+      def valid?
+        token_valid = true
+        now = DateTime.now + Rational(1, 1440)
+        if @id.nil? or now >= @expires_at
+          token_valid = false
+        end
+        token_valid
+      end
+
     end
   end
 end

data/lib/logstash/outputs/monasca/monasca_log_api_client.rb

@@ -1,73 +1,66 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
 require 'rest-client'
 require 'logger'
+require 'singleton'
 
 # relative requirements
 require_relative '../helper/url_helper'
 
-# This class creates a connection to monasca-api
+# This class creates a connection to monasca-log-api
 module LogStash::Outputs
   module Monasca
     class MonascaLogApiClient
 
-      SUPPORTED_API_VERSION = %w(2.0)
+      SUPPORTED_API_VERSION = %w(3.0)
 
       def initialize(host, version)
         @logger = Cabin::Channel.get(LogStash)
-        @rest_client_url = LogStash::Outputs::Helper::UrlHelper.generate_url(host, '/' + check_version(version)).to_s
-        @rest_client = RestClient::Resource.new(@rest_client_url)
+        rest_client_url = LogStash::Outputs::Helper::UrlHelper
+          .generate_url(host, '/' + check_version(version)).to_s
+        @rest_client = RestClient::Resource.new(rest_client_url)
       end
 
-      # Send log events to monasca-api, requires token
-      def send_event(event, data, token, dimensions, application_type=nil)
+      def send_logs(logs, auth_token)
         begin
-          request(event, data, token, dimensions, application_type)
-          @logger.debug("Successfully send event=#{event}, with token=#{token} and dimensions=#{dimensions} to monasca-api")
+          post_header = {
+              'X-Auth-Token' => auth_token,
+              'Content-Type' => 'application/json',
+          }
+          request(logs.to_json, post_header)
         rescue => e
-          @logger.warn('Sending event to monasca-log-api threw exception', :exceptionew => e)
+          @logger.warn('Sending event to monasca-log-api threw exception',
+            :exceptionew => e)
         end
       end
 
       private
 
-      def request(event, data, token, dimensions, application_type)
-        post_headers = {
-            :x_auth_token => token,
-            :content_type => 'application/json',
-        }
-        if dimensions
-          post_headers[:x_dimensions] = dimensions
-        end
-
-        if application_type
-          post_headers[:x_application_type] = application_type
-        end
-
-        @logger.debug('Sending data to ', :url => @rest_client_url)
-        @rest_client['log']['single'].post(data, post_headers)
+      def request(body, header)
+        @logger.debug('Sending data to ', :url => @rest_client.url)
+        @rest_client['logs'].post(body, header)
       end
 
-
       def check_version(version)
         tmp_version = version.sub('v','')
 
         unless SUPPORTED_API_VERSION.include? tmp_version
-          raise "#{tmp_version} is not supported, supported versions are #{SUPPORTED_API_VERSION}"
+          raise "#{tmp_version} is not supported, "\
+            "supported versions are #{SUPPORTED_API_VERSION}"
         end
 
         version