logstash-output-monasca_log_api 0.3.3 → 0.4

data/lib/logstash/outputs/monasca_log_api.rb

@@ -1,26 +1,29 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
 require 'logstash/outputs/base'
 require 'logstash/namespace'
+require 'logstash/codecs/base'
 require 'vine'
+require 'thread'
 
 # relative requirements
 require_relative 'monasca/monasca_log_api_client'
 require_relative 'keystone/keystone_client'
+require_relative 'keystone/token'
 
 # This Logstash Output plugin, sends events to monasca-api.
 # It authenticates against keystone and gets a token.
@@ -28,107 +31,186 @@ require_relative 'keystone/keystone_client'
 class LogStash::Outputs::MonascaLogApi < LogStash::Outputs::Base
   config_name 'monasca_log_api'
 
-  # monasca-api host and port configuration
+  # monasca-log-api configuration
   config :monasca_log_api, :validate => :string, :required => true
-  config :monasca_log_api_version, :validate => :string, :required => false, :default => "v2.0"
+  config :monasca_log_api_version, :validate => :string, :required => false,
+    :default => "v3.0"
 
-  # keystone host and port configuration
+  # keystone configuration
   config :keystone_api, :validate => :string, :required => true
-  # keystone user configuration
   config :project_name, :validate => :string, :required => true
   config :username, :validate => :string, :required => true
   config :password, :validate => :string, :required => true
   config :domain_id, :validate => :string, :required => true
 
-  config :dimensions, :validate => :string, :default => nil
-  config :application_type_key, :validate => :string, :default => 'type'
+  # global dimensions
+  config :dimensions, :validate => :array, :required => false
 
-  attr_accessor :token
+  config :num_of_logs, :validate => :number, :default => 125
+  config :elapsed_time_sec, :validate => :number, :default => 30
+  config :delay, :validate => :number, :default => 10
+  config :max_data_size_kb, :validate => :number, :default => 5120
+
+  attr_accessor :time_thread, :start_time, :logs
 
   default :codec, 'json'
 
+  JSON_LOGS = 'logs'
+  JSON_DIMS = 'dimensions'
+
   public
   def register
-    @keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new keystone_api
-    @monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient.new monasca_log_api, monasca_log_api_version
-    @token = get_token
-
-    @logger.info('Registering keystone user', :username => @username, :project_name => @project_name)
+    @mutex = Mutex.new
+    @logger.info('Registering keystone user',
+      :username => username, :project_name => project_name)
+    @monasca_log_api_client = LogStash::Outputs::Monasca::MonascaLogApiClient
+      .new monasca_log_api, monasca_log_api_version
+    @logs = Hash.new
+    @start_time = nil
+    init_token
+    initialize_logs_object
+    start_time_check
+  end
 
-    @codec.on_event do |event, data|
-      check_token
-      send_event(event, data, dimensions)
+  def multi_receive(events)
+    @logger.debug("Retrieving #{events.size} events")
+    events.each do |event|
+      encode(event)
     end
+  end
 
-  end # def register
+  def close
+    stop_time_check
+  end
 
-  def receive(event)
-    return unless output?(event)
-    if event == LogStash::SHUTDOWN
-      finished
-      return
-    end
-    @codec.encode(event)
-  end # def receive
+  private
 
-  def finished
-    if @shutdown_queue
-      @logger.info('Sending shutdown event to agent queue', :plugin => self.class.config_name)
-      @shutdown_queue << self
-    end
-    if @plugin_state != :finished
-      @logger.info('Plugin is finished', :plugin => self.class.config_name)
-      @plugin_state = :finished
-    end
+  def init_token
+    token = LogStash::Outputs::Keystone::Token.instance
+    token.set_keystone_client(keystone_api)
+    check_token
   end
 
-  def shutdown(queue)
-    teardown
-    @logger.info('Received shutdown signal', :plugin => self.class.config_name)
+  def encode(event)
+    log = generate_log_from_event(event)
+
+    log_bytesize = bytesize_of(log)
+    logs_bytesize = bytesize_of(@logs)
+
+    # if new log would exceed the bytesize then send logs without the new log
+    if @logs[JSON_LOGS] and (logs_bytesize + log_bytesize) > max_data_size_kb
+      @logger.debug("bytesize reached. Sending logs")
+      @mutex.synchronize do
+        send_logs
+        add_log log
+      end
+      return
+
+    # if the new log would reach the maximum bytesize or the maximum allowed
+    # number of sendable logs is reached
+    elsif @logs[JSON_LOGS] and (@logs[JSON_LOGS].size + 1 >= num_of_logs)
+      @logger.debug("bytesize or maximum number of logs reached. Sending logs")
+      @mutex.synchronize do
+        add_log log
+        send_logs
+      end
+      return
 
-    @shutdown_queue = queue
-    if @plugin_state == :finished
-      finished
+    # still free space to collect logs
     else
-      @plugin_state = :terminating
+      @mutex.synchronize do
+        add_log log
+      end
+      return
     end
   end
 
-  private
-  def check_token
-    now = DateTime.now + Rational(1, 1440)
-    if now >= @token.expire_at
-      @token = get_token
-      @logger.info('Token expired. New token requested')
+  def generate_log_from_event(event)
+    message = event.to_hash['message']
+    path = event.to_hash['path']
+    local_dims = JSON.parse(event.to_hash['dimensions'].to_s) if
+      event.to_hash['dimensions']
+    type = event.to_hash['type'] if event.to_hash['type']
+
+    log = { "message" => message, "dimensions" => { "path" => path }}
+    log[JSON_DIMS]['type'] = type if type
+    if local_dims
+      begin
+        JSON.parse(local_dims[0])
+        local_dims.each { |dim|
+          parsed_dim = JSON.parse(dim)
+          log[JSON_DIMS][parsed_dim[0].strip] = parsed_dim[1].strip
+        }
+      rescue
+        log[JSON_DIMS][local_dims[0].strip] = local_dims[1].strip
+      end
     end
+    log
   end
 
-  def send_event(event, data, dimensions)
-    @monasca_log_api_client.send_event(event, data, @token.id, dimensions, get_application_type(event)) if event and @token.id and data
+  def initialize_logs_object
+    if @logs.empty?
+      global_dims = {}
+      dimensions.each { |dim|
+        global_dims[dim.split(':')[0].strip] = dim.split(':')[1].strip
+        } if dimensions
+      @logs = {}
+      @logs[JSON_DIMS] = global_dims unless global_dims.empty?
+      @logs[JSON_LOGS] = []
+    end
   end
 
-  def get_application_type(event)
+  def check_token
+    token = LogStash::Outputs::Keystone::Token.instance
+    token.request_new_token(
+      domain_id, username, password, project_name) unless token.valid?
+  end
 
-    # support for nested.keys.with.lists.0
-    if @application_type_key.include? '.'
-      split_key = @application_type_key.split('.')
-      event_value = event[split_key[0]]
-      access_key = split_key.slice(1..-1).join('.')
+  def start_time_check
+    @time_thread = Thread.new do
 
-      app_type = event_value.access(access_key)
+      loop do
+        unless @start_time.nil?
 
-    else
-      app_type = event[@application_type_key]
+          if @logs[JSON_LOGS] and (@logs[JSON_LOGS].size > 0) and
+            ((Time.now - @start_time) >= elapsed_time_sec)
+            @logger.debug("Time elapsed. Sending logs")
+            @mutex.synchronize do
+              send_logs
+            end
+          end
+
+        end
+        sleep delay
+      end
     end
+  end
 
-    @logger.debug(
-        'Retrieved application_type', :application_type => app_type, :application_type_key => @application_type_key
-    )
-    app_type
+  def stop_time_check
+    @time_thread.kill() if @time_thread
+    @logger.info('Stopped time_check thread')
   end
 
-  def get_token
-    @keystone_client.authenticate(domain_id, username, password, project_name)
+  def bytesize_of(entry)
+    entry.to_json.bytesize / 1024.0
+  end
+
+  def send_logs
+    if @logs[JSON_LOGS] and @logs[JSON_LOGS].size > 0
+      check_token
+      token_id = LogStash::Outputs::Keystone::Token.instance.id
+      @logger.debug("Sending #{@logs[JSON_LOGS].size} logs")
+      @monasca_log_api_client.send_logs(@logs, token_id)
+      @logs.clear
+      initialize_logs_object
+    end
+  end
+
+  def add_log log
+    @logs[JSON_LOGS].push(log)
+    if @logs[JSON_LOGS].size == 1
+      @start_time = Time.now
+    end
   end
 
-end # class LogStash::Outputs::Example
+end

data/logstash-output-monasca_log_api.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-monasca_log_api'
-  s.version = '0.3.3'
+  s.version = '0.4'
   s.licenses = ['Apache License 2.0']
   s.summary = 'This gem is a logstash output plugin to connect via http to monasca-log-api.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'
@@ -19,11 +19,12 @@ Gem::Specification.new do |s|
   s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'output' }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core', '~> 1.5'
-  s.add_runtime_dependency 'logstash-codec-plain', '~> 0.1.6'
-  s.add_runtime_dependency 'logstash-codec-json', '~> 0.1.6'
+  s.add_runtime_dependency 'logstash-core', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-plain', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-json', '~> 2.0'
   s.add_runtime_dependency 'rest-client', '~> 1.8'
   s.add_runtime_dependency 'vine', '~> 0.2'
   s.add_development_dependency 'logstash-devutils', '~> 0.0.14'
   s.add_development_dependency 'simplecov', '~> 0.10'
+  s.add_development_dependency 'rubocop', '~> 0.37.2'
 end

data/spec/outputs/helper/url_helper_spec.rb

@@ -1,16 +1,16 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
@@ -20,14 +20,23 @@ describe LogStash::Outputs::Helper::UrlHelper do
 
   describe ".generate_url" do
     it "generates a URI::HTTP object" do
-      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://192.168.10.5:8080', '/v2.0')).to be_a URI::HTTP
+      expect(LogStash::Outputs::Helper::UrlHelper
+        .generate_url('http://192.168.10.5:8080', '/v2.0')).to be_a URI::HTTP
     end
 
     it "should match a http url" do
-      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://192.168.10.5:8080', '/v2.0').to_s).to eq('http://192.168.10.5:8080/v2.0')
-      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://est.fujitsu:40', nil).to_s).to eq('http://est.fujitsu:40')
-      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('http://est.fujitsu:80', nil).to_s).to eq('http://est.fujitsu')
-      expect(LogStash::Outputs::Helper::UrlHelper.generate_url('https://192.168.10.5:8080', '/v2.0').to_s).to eq('https://192.168.10.5:8080/v2.0')
+      expect(LogStash::Outputs::Helper::UrlHelper
+        .generate_url('http://192.168.10.5:8080', '/v2.0').to_s)
+      .to eq('http://192.168.10.5:8080/v2.0')
+      expect(LogStash::Outputs::Helper::UrlHelper
+        .generate_url('http://est.fujitsu:40', nil).to_s)
+      .to eq('http://est.fujitsu:40')
+      expect(LogStash::Outputs::Helper::UrlHelper
+        .generate_url('http://est.fujitsu:80', nil).to_s)
+      .to eq('http://est.fujitsu')
+      expect(LogStash::Outputs::Helper::UrlHelper
+        .generate_url('https://192.168.10.5:8080', '/v2.0').to_s)
+      .to eq('https://192.168.10.5:8080/v2.0')
     end
   end
 

data/spec/outputs/keystone/keystone_client_spec.rb

@@ -1,16 +1,16 @@
-=begin
-Copyright 2015 FUJITSU LIMITED
-
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
-in compliance with the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software distributed under the License
-is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-or implied. See the License for the specific language governing permissions and limitations under
-the License.
-=end
+# Copyright 2016 FUJITSU LIMITED
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 # encoding: utf-8
 
@@ -18,54 +18,142 @@ require_relative '../spec_helper'
 
 describe LogStash::Outputs::Keystone::KeystoneClient do
 
-  let (:auth_hash) { "{\"auth\":{\"identity\":{\"methods\":[\"password\"],\"password\":{\"user\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi-operator\",\"password\":\"password\"}}},\"scope\":{\"project\":{\"domain\":{\"id\":\"1051bd27b9394120b26d8b08847325c0\"},\"name\":\"csi\"}}}}" }
-
-  let (:ok_response) { stub_response(201, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"token\":{\"methods\":[\"password\"],\"roles\":[{\"id\":\"9fe2ff9ee4384b1894a90878d3e92bab\",\"name\":\"_member_\"},{\"id\":\"4e9ef1ffe73446c6b02f8fce0585c307\",\"name\":\"monasca-user\"}],\"expires_at\":\"2015-05-26T08:55:36.774122Z\",\"project\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"1051bd27b9394120b26d8b08847325c0\",\"name\":\"mini-mon\"},\"user\":{\"domain\":{\"id\":\"default\",\"name\":\"Default\"},\"id\":\"06ecc9869b8e4846b2fce3e5759ba4af\",\"name\":\"mini-mon\"},\"audit_ids\":[\"ORap7R56S2S-p6tFVeMkpg\"],\"issued_at\":\"2015-05-26T07:55:36.774146Z\"}}") }
-
-  let (:failed_response) { stub_response(401, {:x_subject_token => "f8cdafb7dce94444ad781a53ddaff693"}, "{\"error\": {\"message\": \"Could not find project: f8cdafb7dce94444ad781a53ddaff693 (Disable debug mode to suppress these details.)\", \"code\": 401, \"title\": \"Unauthorized\"}}") }
+  let (:valid_date) { DateTime.now + Rational(1, 1440) }
+  let (:token) { "f8cdafb7dce94444ad781a53ddaff693" }
+  let (:valid_token) { {:token => token,
+    :expires_at => valid_date } }
+  let (:domain_id) { "default" }
+  let (:username) { "operator" }
+  let (:password) { "zPrHGl<IJtn=ux;{&T/nfXh=H" }
+  let (:project_name) { "monasca" }
+  let (:auth_hash) {
+    {
+      "auth"=>{
+        "identity"=>{
+          "methods"=>["password"],
+          "password"=>{
+            "user"=>{
+              "domain"=>{"id"=>domain_id},
+              "name"=>username,
+              "password"=>password
+            }
+          }
+        },
+        "scope"=>{
+          "project"=>{
+            "domain"=>{"id"=>domain_id},
+            "name"=>project_name
+          }
+        }
+      }
+    }.to_json
+  }
+
+  let (:ok_response) { stub_response(201,
+    {:x_subject_token => token},
+    {
+      "token"=>{
+        "methods"=>["password"],
+        "roles"=>[
+          {
+            "id"=>"9fe2ff9ee4384b1894a90878d3e92bab",
+            "name"=>"_member_"
+          },
+          {
+            "id"=>"4e9ef1ffe73446c6b02f8fce0585c307",
+            "name"=>"monasca-user"
+          }
+        ],
+        "expires_at"=>valid_date.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "project"=>{
+          "domain"=>{
+            "id"=>"default",
+            "name"=>"Default"
+          },
+          "id"=>"1051bd27b9394120b26d8b08847325c0",
+          "name"=>project_name
+        },
+        "user"=>{
+          "domain"=>{
+            "id"=>"default",
+            "name"=>"Default"
+          },
+          "id"=>"06ecc9869b8e4846b2fce3e5759ba4af",
+          "name"=>username
+        },
+        "audit_ids"=>["ORap7R56S2S-p6tFVeMkpg"],
+        "issued_at"=>"2015-05-26T07:55:36.774146Z"
+      }
+    }.to_json)
+  }
+
+  let (:failed_response) { stub_response(401,
+    {:x_subject_token => ""},
+    {
+      "error"=>{
+        "message"=>"Could not find project: f8cdafb7dce94444ad781a53ddaff693 "\
+          "(Disable debug mode to suppress these details.)",
+        "code"=>401,
+        "title"=>"Unauthorized"
+      }
+    }.to_json)
+  }
 
   context 'when initializing' do
     it 'then it should register without exceptions' do
-      expect {LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')}.to_not raise_error
+      expect {LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')}
+        .to_not raise_error
     end
 
     it "returns a failure if arguments are missing" do
-      expect {LogStash::Outputs::Keystone::KeystoneClient.new}.to raise_exception(ArgumentError)
+      expect {LogStash::Outputs::Keystone::KeystoneClient.new}
+        .to raise_exception(ArgumentError)
     end
   end
 
   context 'when authenticates' do
     it 'then it should request to keystone' do
       expect_any_instance_of(RestClient::Resource).to receive(:post)
-        .with(auth_hash, :content_type => 'application/json', :accept => 'application/json')
-      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
-      allow(keystone_client).to receive(:handle_response).and_return(LogStash::Outputs::Keystone::Token.new('abc', DateTime.now + Rational(5, 1440)))
-      keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+        .with(auth_hash, :content_type => 'application/json',
+          :accept => 'application/json').and_return(ok_response)
+
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient
+        .new('hostname:8080')
+      keystone_client.authenticate(domain_id, username, password, project_name)
     end
 
-    it 'then it should create a new token' do
-      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
-      allow(keystone_client).to receive(:request).and_return(ok_response)
-      expected = LogStash::Outputs::Keystone::Token.new('f8cdafb7dce94444ad781a53ddaff693', DateTime.parse("2015-05-26T08:55:36.774122Z"))
-      actual = keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
-      expect(actual).to eq(expected)
+    it 'then it should return a token' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .and_return(ok_response)
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient
+        .new('hostname:8080')
+      token = keystone_client
+        .authenticate(domain_id, username, password, project_name)
+      expect(token["token"]).to eq(valid_token["token"])
+      expect(token["expires_at"].to_s).to eq(valid_token["expires_at"].to_s)
     end
   end
 
   context 'when authentication failed' do
-    it 'then it should not create a new token' do
+    it 'then it should log an information' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .and_return(failed_response)
       expect_any_instance_of(Cabin::Channel).to receive(:info)
-      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
-      allow(keystone_client).to receive(:request).and_return(failed_response)
-      actual = keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
-      expect(actual).to eq(nil)
+
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient
+        .new('hostname:8080')
+      keystone_client.authenticate(domain_id, username, password, project_name)
     end
 
-    it 'then it should log an information' do
+    it 'then it should return nil' do
+      expect_any_instance_of(RestClient::Resource).to receive(:post)
+        .and_return(failed_response)
       expect_any_instance_of(Cabin::Channel).to receive(:info)
-      keystone_client = LogStash::Outputs::Keystone::KeystoneClient.new('hostname:8080')
-      allow(keystone_client).to receive(:request).and_return(failed_response)
-      keystone_client.authenticate('1051bd27b9394120b26d8b08847325c0', 'csi-operator', 'password', 'csi')
+
+      keystone_client = LogStash::Outputs::Keystone::KeystoneClient
+        .new('hostname:8080')
+      expect(keystone_client
+        .authenticate(domain_id, username, password, project_name)).to eq(nil)
     end
   end