fluent-plugin-cloudstack 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c162238913dc3df257fced1b4511542da70bcc8
+  data.tar.gz: ff71b4d47e21628a944fc89626ff2c2d069a7467
+SHA512:
+  metadata.gz: 5e73c124caa441d3453bf58143d466bae661dcc906c721d0ab8b4ad1b0547300db7c57e72a654876532b2a3e86ced2c2b0979c0094ca2f35d58456376fb2614a
+  data.tar.gz: c31c87c1f4d276742f9de37d67df857f6a9642188f8079f1b9622cf516227c77e4a50baacbcd6af8dd7eec5acb9c88394184ffc261e1a82ad6bb14a61ee641c0

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/
+vendor/*
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gemspec
+

data/Gemfile.lock

@@ -0,0 +1,57 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-cloudstack (0.0.1)
+      eventmachine
+      fluentd (>= 0.10.7)
+      fog
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    cool.io (1.1.1)
+      iobuffer (>= 1.0.0)
+    eventmachine (1.0.3)
+    excon (0.27.6)
+    fluentd (0.10.39)
+      cool.io (~> 1.1.1)
+      http_parser.rb (~> 0.5.1)
+      json (>= 1.4.3)
+      msgpack (>= 0.4.4, < 0.6.0, != 0.5.3, != 0.5.2, != 0.5.1, != 0.5.0)
+      yajl-ruby (~> 1.0)
+    fog (1.16.0)
+      builder
+      excon (~> 0.27.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5)
+      ruby-hmac
+      unicode (~> 0.4.4)
+    formatador (0.2.4)
+    http_parser.rb (0.5.3)
+    iobuffer (1.1.2)
+    json (1.8.1)
+    mime-types (1.25)
+    mini_portile (0.5.2)
+    msgpack (0.5.6)
+    multi_json (1.8.2)
+    net-scp (1.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.7.0)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    rake (10.1.0)
+    ruby-hmac (0.4.0)
+    unicode (0.4.4)
+    yajl-ruby (1.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fluent-plugin-cloudstack!
+  rake

data/README.textile

@@ -0,0 +1,83 @@
+h1. Fluent::Plugin::CloudStack
+
+Fluentd input plugin to get usages and events from CloudStack API.
+
+* usages
+** instance_usage
+** memory_usage
+** root_volume_usage
+** data_volume_usage
+** usages_per_service_offering
+** usages_per_disk_offering
+* event
+
+
+h2. Installation
+
+Add this line to your application's Gemfile:
+
+<pre>
+gem 'fluent-plugin-cloudstack'
+</pre>
+
+Or install it yourself as:
+
+<pre>
+$ gem install fluent-plugin-cloudstack
+</pre>
+
+Or use td-agent : (on Ubuntu12.04)
+
+<pre>
+$ sudo /usr/lib/fluent/ruby/bin/fluent-gem install fluent-plugin-cloudstack
+</pre>
+
+
+
+h2. Configuration
+
+<pre>
+<source>
+  type cloudstack
+  host $cloudtack_host
+  apikey $cloudstack_apikey
+  secretkey $cloustack_secretkey
+
+  # optional
+  protocol $cloudstack_protocol_scheme # https or http, default https
+  path $cloudstack_path                # default /client/api
+  port $cloudstack_port                # default 443
+  interval $get_interval_sec           # min 300, default 300
+  ssl $cloudtack_api_ssl               # true or false, default true
+  domain_id $cloudstack_domain_id
+
+  tag $fluentd_tag                     # default cloudstack
+</source>
+</pre>
+
+
+
+h2. Data Example
+
+<pre>
+20131027T193928+0000    cloudstack.usages      {"instances_usage":1,
+                                                "memory_usage":512,
+                                                "cpu_usage":1,
+                                                "root_volume_usage":16106127360,
+                                                "data_volume_usage":42949672960,
+                                                "usages_per_service_offering":{"t1.micro":1},
+                                                "usages_per_disk_offering":{"Datadisk 40GB":1}}
+
+20130507T041336+0000    cloudstack.event       {"id":"xxxxxxx-4e64-4fa5-8045-5ecffb7e2417",
+                                                "username":"clnxxxxxxxxx",
+                                                "type":"USER.LOGIN",
+                                                "level":"INFO",
+                                                "description":"user has logged in from IP Address xxx.xxx.xxx.xxx",
+                                                "account":"clnxxxxxxx",
+                                                "domainid":"xxxxxxxx-dcc3-486b-90e8-02c514c93d1f",
+                                                "domain":"cln10000004",
+                                                "created":"2013-05-07T13:13:36+0900",
+                                                "state":"Completed",
+                                                "parentid":"0"}
+</pre>
+

data/Rakefile

@@ -0,0 +1,15 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.test_files  = Dir['test/plugin/*.rb']
+  test.ruby_opts = ['-rubygems'] if defined? Gem
+  test.ruby_opts << '-I.'
+  test.verbose = true
+end
+
+task :default => :test
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/fluent-plugin-cloudstack.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-cloudstack"
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Yuichi UEMURA"]
+  gem.email       = ["yuichi.u@gmail.com"]
+  gem.homepage    = "https://github.com/u-ichi/fluent-plugin-cloudstack"
+  gem.summary     = %q{cloudstack plugin for fluentd, a log collector}
+  gem.description = %q{cloudstack plugin for fluentd, a log collector}
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency "fluentd", ">= 0.10.7"
+  gem.add_dependency "fog"
+  gem.add_dependency "eventmachine"
+
+  gem.add_development_dependency "rake"
+end
+

data/lib/fluent/plugin/in_cloudstack.rb

@@ -0,0 +1,175 @@
+module Fluent
+  class CloudStackInput < Input
+    Fluent::Plugin.register_input("cloudstack", self)
+
+    INTERVAL_MIN = 300
+
+    config_param :host
+    config_param :path, :default =>'/client/api'
+    config_param :protocol, :default =>'https'
+    config_param :port, :default =>443
+    config_param :apikey
+    config_param :secretkey
+    config_param :domain_id, :default => nil
+    config_param :interval, :default => INTERVAL_MIN
+
+    config_param :tag, :string, :default => "cloudstack"
+    config_param :ssl, :bool, :default => true
+
+    attr_writer :before_events
+
+    def initialize
+      require "fog"
+      require "eventmachine"
+      init_eventmachine
+
+      @before_events_filepath = "logs/before_events.yml"
+
+      if File.exist?(@before_events_filepath)
+        @before_events = YAML.load_file(@before_events_filepath)
+      else
+        @before_events = nil
+      end
+
+      super
+    end
+
+    def configure(conf)
+      super
+      @conf = conf
+      unless @host && @apikey && @secretkey
+        raise ConfigError, "'host' and 'apikey' and 'secretkey' must be all specified."
+      end
+
+      if @interval < INTERVAL_MIN
+        raise ConfigError, "'interval' must be over #{INTERVAL_MIN}."
+      end
+    end
+
+    def start
+      super
+      run_reactor_thread
+      @thread = Thread.new(&method(:run))
+      $log.info "listening cloudstack api on #{@host}"
+    end
+
+    def shutdown
+      super
+      @thread.join
+      EM.stop if EM.reactor_running?
+      @reactor_thread.join if @reactor_thread
+    end
+
+    def run
+      EM.add_periodic_timer(@interval) do
+        emit_new_events
+        emit_usages
+      end
+    end
+
+    def emit_new_events
+      new_events = get_new_events
+      output_tag = "#{@tag}.event"
+      new_events.each do |event|
+        time = Time.parse(event["created"]).to_i
+        Engine.emit(output_tag, time, event)
+      end
+    end
+
+    def emit_usages
+      usages = get_usages
+      time = Engine.now
+      output_tag = "#{@tag}.usages"
+      Engine.emit(output_tag, time, usages)
+    end
+
+    def get_new_events
+      if @before_events
+        startdate = Time.parse(@before_events[0]["created"])
+        event_responses = cs.list_events(:startdate => startdate.strftime("%Y-%m-%d %H:%M:%S"), :domainid => @domain_id)
+        events = Array.new
+        event_responses["listeventsresponse"]["event"].each do |event|
+          unless Time.parse(event["created"]) == startdate
+            events.push event
+          end
+        end
+      else
+        events = cs.list_events(:domainid => @domain_id)["listeventsresponse"]["event"]
+      end
+
+      if events.size > 0
+        File.write(@before_events_filepath, events.to_yaml)
+        @before_events = events
+      end
+
+      events
+    end
+
+    def get_usages
+      usages_per_service_offering   = Hash.new(0)
+      usages_per_disk_offering      = Hash.new(0)
+      memory_usage      = 0
+      cpu_usage         = 0
+      root_volume_usage = 0
+      data_volume_usage = 0
+
+      vms_responses = cs.list_virtual_machines(:domainid=>@domain_id)
+      vms =  vms_responses["listvirtualmachinesresponse"]["virtualmachine"]
+      vms.each do |vm|
+        memory_usage += vm["memory"].to_i
+        cpu_usage += vm["cpunumber"].to_i
+        usages_per_service_offering[vm["serviceofferingname"]] += 1
+      end
+
+      volumes_responses = cs.list_volumes(:domainid=>@domain_id)
+      volumes =  volumes_responses["listvolumesresponse"]["volume"]
+      volumes.each do |volume|
+        case volume["type"]
+        when "ROOT"
+          root_volume_usage += volume["size"]
+        when "DATADISK"
+          data_volume_usage += volume["size"]
+          usages_per_disk_offering[volume["diskofferingname"]] += 1
+        end
+      end
+
+      results =  {:vm_usage             => vms.size,
+                  :memory_usage                => memory_usage,
+                  :cpu_usage                   => cpu_usage,
+                  :root_volume_usage           => root_volume_usage,
+                  :data_volume_usage           => data_volume_usage,
+                  :usages_per_service_offering => usages_per_service_offering,
+                  :usages_per_disk_offering    => usages_per_disk_offering,
+      }
+    end
+
+    def cs
+      @@cs ||= Fog::Compute.new(
+          :provider => 'CloudStack',
+          :cloudstack_api_key           => @apikey,
+          :cloudstack_secret_access_key => @secretkey,
+          :cloudstack_host              => @host,
+          :cloudstack_port              => @port,
+          :cloudstack_path              => @path,
+          :cloudstack_scheme            => @protocol,
+      )
+    end
+
+    private
+
+    def run_reactor_thread
+      unless EM.reactor_running?
+        @reactor_thread = Thread.new { EM.run }
+      end
+    end
+
+    def init_eventmachine
+      EM.epoll; EM.kqueue
+      EM.error_handler do |ex|
+        $log.error("Eventmachine problem")
+        $log.error("#{ex}, tracelog : \n#{ex.backtrace.join("\n")}")
+      end
+    end
+  end
+end
+

data/test/plugin/in_cloudstack.rb

@@ -0,0 +1,48 @@
+require 'test/unit'
+require 'test_helper'
+require 'lib/fluent/plugin/in_cloudstack.rb'
+require 'pp'
+
+
+class CloudStackInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    host localhost
+    apikey hoge
+    secretkey fuga
+    domain_id domain_id
+  ]
+
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::CloudStackInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'localhost',      d.instance.host
+    assert_equal 'hoge',           d.instance.apikey
+    assert_equal 'fuga',           d.instance.secretkey
+    assert_equal 'domain_id',      d.instance.domain_id
+  end
+
+
+  def test_get_events
+    d = create_driver
+
+    # d.instance.before_events = before_events_stub
+  end
+
+  def test_get_usage
+    d = create_driver
+
+    # d.instance.get_usages
+  end
+
+end
+
+
+

data/test/test_helper.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-cloudstack
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Yuichi UEMURA
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.7
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: cloudstack plugin for fluentd, a log collector
+email:
+- yuichi.u@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.textile
+- Rakefile
+- VERSION
+- fluent-plugin-cloudstack.gemspec
+- lib/fluent/plugin/in_cloudstack.rb
+- test/plugin/in_cloudstack.rb
+- test/test_helper.rb
+homepage: https://github.com/u-ichi/fluent-plugin-cloudstack
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.10
+signing_key: 
+specification_version: 4
+summary: cloudstack plugin for fluentd, a log collector
+test_files:
+- test/plugin/in_cloudstack.rb
+- test/test_helper.rb