fluent-plugin-azuremonitorlog 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bc27eaae665cf1497fcb31b343eab908af3d48a8
+  data.tar.gz: ffaa0adb94b8ca19d7c880afc78d78b41dab8846
+SHA512:
+  metadata.gz: a856ecc2e3208d0d1b17dfba8727c8dd846dc33a54dd961ff90479bf61f865bf2b1afc6b7344d55f2916c9bd64a8ac8284d99ca852597482b64c4e241caf49ea
+  data.tar.gz: 23f60100301a8a6d9cee11aeb42a488a3635e05c607d20cba64bac299b06cc1a753b57bf02471db6cec4d2a5b1586ca1fc74956587d19919a1ce4a0fcc4aabd8

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-azuremonitor.gemspec
+gemspec

data/Gemfile.fluentd.0.12

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.12.0'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-azuremonitorlog (0.0.2)
+      azure_mgmt_monitor (~> 0.11.0)
+      fluentd (>= 0.10.30)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    azure_mgmt_monitor (0.11.0)
+      ms_rest_azure (~> 0.8.0)
+    concurrent-ruby (1.0.5)
+    cool.io (1.5.1)
+    domain_name (0.5.20170404)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (0.13.1)
+      multipart-post (>= 1.2, < 3)
+    faraday-cookie_jar (0.0.6)
+      faraday (>= 0.7.4)
+      http-cookie (~> 1.0.0)
+    fluentd (0.14.21)
+      cool.io (>= 1.4.5, < 2.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      ruby_dig (~> 0.0.2)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (~> 0.1.7)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    http_parser.rb (0.6.0)
+    ms_rest (0.7.1)
+      concurrent-ruby (~> 1.0)
+      faraday (~> 0.9)
+      timeliness (~> 0.3)
+    ms_rest_azure (0.8.2)
+      concurrent-ruby (~> 1.0)
+      faraday (~> 0.9)
+      faraday-cookie_jar (~> 0.0.6)
+      ms_rest (~> 0.7.0)
+    msgpack (1.1.0)
+    multipart-post (2.0.0)
+    power_assert (1.1.0)
+    rake (12.0.0)
+    ruby_dig (0.0.2)
+    serverengine (2.0.5)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.1.9)
+    test-unit (3.2.5)
+      power_assert
+    thread_safe (0.3.6)
+    timeliness (0.3.8)
+    tzinfo (1.2.3)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2017.2)
+      tzinfo (>= 1.0.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.4)
+    yajl-ruby (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-azuremonitorlog!
+  rake (>= 0.9.2)
+  test-unit (>= 3.1.0)
+
+BUNDLED WITH
+   1.15.4

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2017 Microsoft
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# fluent-plugin-azuremonitorlog, a plugin for [Fluentd](http://fluentd.org) 
+## Overview
+
+***Azure Monitor log*** input plugin.
+
+This plugin gets the monitor activity logs from Azure Monitor API to fluentd.
+
+## Installation
+
+Install from RubyGems:
+```
+$ gem install fluent-plugin-azuremonitorlog
+```
+
+To use this plugin, you need to have Azure Service Principal.<br/>
+Create an Azure Service Principal through [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/create-an-azure-service-principal-azure-cli?toc=%2fazure%2fazure-resource-manager%2ftoc.json) or [Azure portal](https://docs.microsoft.com/en-us/azure/azure-resource-manager/resource-group-create-service-principal-portal).
+
+## Configuration
+
+```config
+<source>
+  @type azuremonitorlog
+  tag azuremonitorlog
+  tenant_id [Azure_Tenant_ID]
+  subscription_id [Azure_Subscription_Id]
+  client_id [Azure_Client_Id]
+  client_secret [Azure_Client_Secret]
+  
+  select [selected fields to query]
+  filter [filter the query query] (default: eventChannels eq 'Admin, Operation')
+  interval [interval in seconds] (default: 300)
+  api_version [api version] (default: 2015-04-01)
+</source>
+```
+
+Documentation for select and filter can be found [here](https://docs.microsoft.com/en-us/rest/api/monitor/ActivityLogs/List#activitylogs_list_uri_parameters)
+
+### Example for source config
+
+```config
+<source>
+    @type azuremonitorlog
+    tag azuremonitorlog
+    tenant_id [Azure_Tenant_ID]
+    subscription_id [Azure_Subscription_Id]
+    client_id [Azure_Client_Id]
+    client_secret [Azure_Client_Secret]
+    select_filter eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId,submissionTimestamp,level
+</source>
+
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require "rake/testtask"
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-azuremonitorlog.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-azuremonitorlog"
+  gem.version       = "0.0.1"
+  gem.authors       = ["Ilana Kantorov"]
+  gem.email         = ["ilanak@microsoft.com"]
+  gem.description   = %q{Input plugin for Azure Monitor Activity logs.}
+  gem.homepage      = "https://github.com/Ilanak/fluent-plugin-azureamonitorlog"
+  gem.summary       = gem.description
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "fluentd", ">= 0.10.30"
+  gem.add_dependency "azure_mgmt_monitor", "~> 0.11.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.1.0"
+  gem.license = 'MIT'
+end

data/lib/fluent/plugin/in_azuremonitorlog.rb

@@ -0,0 +1,132 @@
+require 'fluent/input'
+require 'azure_mgmt_monitor'
+require 'uri'
+
+class Fluent::AzureMonitorLogInput < Fluent::Input
+  Fluent::Plugin.register_input("azuremonitorlog", self)
+
+  # To support log_level option implemented by Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  config_param :tag,                :string
+  config_param :tenant_id,          :string, :default => nil
+  config_param :subscription_id,    :string, :default => nil
+  config_param :client_id,          :string, :default => nil
+  config_param :client_secret,      :string, :default => nil, :secret => true
+
+  config_param :select,             :string, :default => nil
+  config_param :filter,             :string, :default => "eventChannels eq 'Admin, Operation'"
+  config_param :interval,           :integer,:default => 300
+  config_param :api_version,        :string, :default => '2015-04-01'
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+
+    provider = MsRestAzure::ApplicationTokenProvider.new(@tenant_id, @client_id, @client_secret)
+    credentials = MsRest::TokenCredentials.new(provider)
+    @client = Azure::ARM::Monitor::MonitorManagementClient.new(credentials);
+    @client.subscription_id = @subscription_id
+  end
+
+  def start
+    super
+    @watcher = Thread.new(&method(:watch))
+  end
+
+  def shutdown
+    super
+    @watcher.terminate
+    @watcher.join
+  end
+
+  def set_query_options(filter, custom_headers)
+    fail ArgumentError, 'path is nil' if @client.subscription_id.nil?
+
+    request_headers = {}
+
+    # Set Headers
+    request_headers['x-ms-client-request-id'] = SecureRandom.uuid
+    request_headers['accept-language'] = @client.accept_language unless @client.accept_language.nil?
+
+    {
+        middlewares: [[MsRest::RetryPolicyMiddleware, times: 3, retry: 0.02], [:cookie_jar]],
+        path_params: {'subscriptionId' => @client.subscription_id},
+        query_params: {'api-version' => @api_version, '$filter' => filter, '$select' => @select},
+        headers: request_headers.merge(custom_headers || {}),
+        base_url: @client.base_url
+    }
+  end
+
+  private
+
+  def watch
+    while true
+        log.debug "azure monitorlog: watch thread starting"
+        output
+        sleep @interval
+    end
+  end
+
+  def output
+      start_time = Time.now - @interval
+      end_time = Time.now
+      
+      log.debug "start time: #{start_time}, end time: #{end_time}"
+      filter = "eventTimestamp ge '#{start_time}' and eventTimestamp le '#{end_time}'"
+
+      if !@filter.empty?
+        filter += " and #{@filter}"
+      end
+
+      monitor_logs_promise = get_monitor_log_async(filter)
+      monitor_logs = monitor_logs_promise.value!
+
+      if !monitor_logs.body['value'].nil? and  monitor_logs.body['value'].any?
+        monitor_logs.body['value'].each {|val|
+          router.emit(@tag, Time.now.to_i, val)
+        }
+      else
+        log.debug "empty"
+      end
+  end
+
+  def get_monitor_log_async(filter = nil, custom_headers = nil)
+    options = set_query_options(filter, custom_headers)
+    path_template = '/subscriptions/{subscriptionId}/providers/microsoft.insights/eventtypes/management/values'
+    promise = @client.make_request_async(:get, path_template, options)
+
+    promise = promise.then do |result|
+      http_response = result.response
+      status_code = http_response.status
+      response_content = http_response.body
+      unless status_code == 200
+        error_model = JSON.load(response_content)
+        log.error(error_model['error']['message'])
+      end
+
+      result.request_id = http_response['x-ms-request-id'] unless http_response['x-ms-request-id'].nil?
+      # Deserialize Response
+      if status_code == 200
+        begin
+          result.body = response_content.to_s.empty? ? nil : JSON.load(response_content)
+        rescue Exception => e
+          log.error("Error occurred in parsing the response")
+        end
+      end
+
+      result
+    end
+
+    promise.execute
+  end
+end

data/test/helper.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'bundler'
+require 'fluent/input'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require "test/unit"
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require "fluent/test"
+unless ENV.has_key?("VERBOSE")
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      #pass
+    end
+  }
+  $log = nulllogger
+end
+
+require "fluent/plugin/in_azuremonitorlog"
+
+class Test::Unit::TestCase
+end
+

data/test/plugin/test_in_azuremonitorlog.rb

@@ -0,0 +1,46 @@
+require 'helper'
+
+class AzureMonitorLogInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  ### for monitor log
+  CONFIG_MONITOR_LOG = %[
+    tag azuremonitorlog
+    tenant_id test_tenant_id
+    subscription_id test_subscription_id
+    client_id test_client_id
+    client_secret test_client_secret
+    select eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId
+    filter eventChannels eq 'Admin, Operation'
+    interval 300
+    api_version 2015-04-01
+  ]
+
+  def create_driver_monitor_log(conf = CONFIG_MONITOR_LOG)
+    Fluent::Test::InputTestDriver.new(Fluent::AzureMonitorLogInput).configure(conf)
+  end
+
+  def test_configure_monitor_log
+    d = create_driver_monitor_log
+    assert_equal 'azuremonitorlog', d.instance.tag
+    assert_equal 'test_tenant_id', d.instance.tenant_id
+    assert_equal 'test_subscription_id', d.instance.subscription_id
+    assert_equal 'test_client_id', d.instance.client_id
+    assert_equal 'test_client_secret', d.instance.client_secret
+    assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', d.instance.select
+    assert_equal 'eventChannels eq \'Admin, Operation\'', d.instance.filter
+    assert_equal 300, d.instance.interval
+    assert_equal '2015-04-01', d.instance.api_version
+  end
+
+  def test_set_query_options
+    d = create_driver_monitor_log
+    query_options = d.instance.set_query_options(d.instance.filter, {})
+    assert_equal '2015-04-01', query_options[:query_params]['api-version']
+    assert_equal 'eventChannels eq \'Admin, Operation\'', query_options[:query_params]['$filter']
+    assert_equal 'eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId', query_options[:query_params]['$select']
+  end
+
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-azuremonitorlog
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ilana Kantorov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.30
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.30
+- !ruby/object:Gem::Dependency
+  name: azure_mgmt_monitor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+description: Input plugin for Azure Monitor Activity logs.
+email:
+- ilanak@microsoft.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.fluentd.0.12
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-azuremonitorlog.gemspec
+- lib/fluent/plugin/in_azuremonitorlog.rb
+- test/helper.rb
+- test/plugin/test_in_azuremonitorlog.rb
+homepage: https://github.com/Ilanak/fluent-plugin-azureamonitorlog
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Input plugin for Azure Monitor Activity logs.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_azuremonitorlog.rb