oneacct-export 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 670e8800aaf7a64b5f75d0f6d930e0d9be4d96fb
+  data.tar.gz: e6545c409b1c10e466a927833d3ffcde5b0c9c19
+SHA512:
+  metadata.gz: 79f668da9123a2b2d83eb3131062494614ce605132b394921b0867faeb72c53c7a16649095e31928f6e59eb0163b4e78574670515cd928dfbaf75914bdf708bc
+  data.tar.gz: fe9e971c3b06d3f7717e24feca0379206f4e8123276d01815618c475b331fbcf2e30db1bad2cbfa19b0190615ff3d759950d2f097c25bd9e913596f15c822fa2

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.gem
+*.swp
+*.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in oneacct_export.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,11 @@
+# OneacctExport
+
+Exporting OpenNebula accounting data.
+
+## Contributing
+
+1. Fork it ( https://github.com/Misenko/oneacct_export/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,18 @@
+require 'bundler/gem_tasks'
+require 'rubygems/tasks'
+
+task :default => 'test'
+
+desc "Run all tests; includes rspec and coverage reports"
+task :test => 'rcov:rspec'
+
+desc "Run all tests; includes rspec and coverage reports"
+task :spec => 'test'
+
+Gem::Tasks.new(:build => {:tar => true, :zip => true}, :sign => {:checksum => true, :pgp => false})
+
+namespace :rcov do
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:rspec)
+end

data/bin/oneacct-export

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+ENV['RAILS_ENV'] = ENV['RAILS_ENV'] ? ENV['RAILS_ENV'] : 'production'
+
+require 'syslogger'
+require 'oneacct_exporter'
+require 'oneacct_exporter/log'
+require 'settings'
+require 'fileutils'
+require 'oneacct_opts'
+
+options = OneacctOpts.parse(ARGV)
+
+log = Logger.new(STDOUT)
+
+if Settings['logging'] && Settings['logging']['log_type'] == 'file'
+  begin
+    log_file = File.open(Settings['logging']['log_file'], File::WRONLY | File::CREAT | File::APPEND)
+    log = Logger.new(log_file)
+  rescue => e
+    OneacctExporter::Log.setup_logging(log)
+    log.warn("Unable to create log file #{Settings['logging']['log_file']}: #{e.message}.\
+           Falling back to STDOUT.")
+  end
+elsif Settings['logging'] && Settings['logging']['log_type'] == 'syslog'
+  log = Syslogger.new('oneacct-export')
+end
+
+OneacctExporter::Log.setup_log_level(log)
+
+range = {}
+range[:from] = options.records_from
+range[:to] = options.records_to
+
+groups = {}
+groups[:include] = options.include_groups if options.include_groups
+groups[:exclude] = options.exclude_groups if options.exclude_groups
+
+if options.groups_file
+  log.debug('Reading groups from file...')
+  if File.exist?(options.groups_file) && File.readable?(options.groups_file)
+    file = File.open(options.groups_file, 'r')
+    file.each_line do |line|
+      groups[groups.keys.first] << line
+    end
+    file.close
+  else
+    log.error("File contaning groups: #{options.groups_file} doesn't exists or cannot be read. "\
+              'Skipping groups restriction...')
+    groups[groups.keys.first] = []
+  end
+end
+
+begin
+  FileUtils.mkdir_p Settings.output['output_dir']
+rescue SystemCallError => e
+  puts "Cannot create an output directory: #{e.message}. Quitting."
+  exit
+end
+
+log.debug('Creating OneacctExporter...')
+
+opts = {}
+opts[:range] = range
+opts[:groups] = groups
+opts[:blocking] = options.blocking
+opts[:timeout] = options.timeout
+opts[:compatibility] = options.compatibility
+
+log.debug(opts)
+
+oneacct_exporter = OneacctExporter.new(opts, log)
+oneacct_exporter.export

data/config/conf.yml

@@ -0,0 +1,53 @@
+#Five mandatory parameters: 
+# site_name
+# cloud_type
+# endpoint
+# output -> output_dir
+# output -> output_type
+defaults: &defaults
+  site_name: <fill in> 
+  cloud_type: <fill in>
+  endpoint: <fill in>
+  output:
+    output_dir: <fill in>
+    output_type: apel-0.2 #currently only option
+# num_of_vms_per_file: 500
+#
+#logging:
+# log_type: file #Two options: file, syslog. Defaults to stdout
+# log_file: /var/log/oneacct-export.log #Used when type file selected
+#
+#xml_rpc:
+# secret: username:password #If not specified looking for secret in ONE_AUTH and ~/.one/one_auth
+# endpoint: http://localhost:2633/RPC2 #Defaults to content of ONE_XMLRPC or content of ~/.one/one_endpoint or http://localhost:2633/RPC2
+#
+#redis:
+# namespace: oneacct_export
+# url: redis://localhost:6379
+# password: password
+  sidekiq:
+    queue: oneacct_export
+production:
+  <<: *defaults
+development:
+  <<: *defaults
+test:
+  site_name: <placeholder>
+  cloud_type: <placeholder>
+  endpoint: <placeholder>
+  output:
+    output_dir: <placeholder>  
+    output_type: <placeholder>
+    num_of_vms_per_file: <placeholder>
+  logging:
+    log_type: <placeholder>
+    log_file: <placeholder>
+  xml_rpc:
+    secret: <placeholder>
+    endpoint: <placeholder>
+  redis:
+    namespace: <placeholder>
+    url: http://some.random.url.com
+    password: password
+  sidekiq:
+    queue: oneacct_export

data/config/sidekiq.yml

@@ -0,0 +1,6 @@
+---
+:concurrency: 5
+#:pidfile: ./tmp/pids/sidekiq.pid
+#:logfile: ./log/sidekiq.log
+:queues:
+    - oneacct_export

data/lib/errors.rb

@@ -0,0 +1,7 @@
+require 'errors/authentication_error'
+require 'errors/resource_not_found_error'
+require 'errors/resource_retrieval_error'
+require 'errors/resource_state_error'
+require 'errors/user_not_authorized_error'
+
+module Errors; end

data/lib/errors/authentication_error.rb

@@ -0,0 +1,3 @@
+module Errors
+  class AuthenticationError < ::StandardError; end
+end

data/lib/errors/resource_not_found_error.rb

@@ -0,0 +1,3 @@
+module Errors
+  class ResourceNotFoundError < ::ArgumentError; end
+end

data/lib/errors/resource_retrieval_error.rb

@@ -0,0 +1,3 @@
+module Errors
+  class ResourceRetrievalError < ::StandardError; end
+end

data/lib/errors/resource_state_error.rb

@@ -0,0 +1,3 @@
+module Errors
+  class ResourceStateError < ::StandardError; end
+end

data/lib/errors/user_not_authorized_error.rb

@@ -0,0 +1,3 @@
+module Errors
+  class UserNotAuthorizedError < AuthenticationError; end
+end

data/lib/input_validator.rb

@@ -0,0 +1,18 @@
+require 'uri'
+
+module InputValidator
+  URI_RE = /\A#{URI.regexp}\z/
+  NUMBER_RE = /\A[[:digit:]]+\z/
+
+  def is?(object, regexp)
+    object.to_s =~ regexp
+  end
+
+  def is_number?(object)
+    is?(object, NUMBER_RE)
+  end
+
+  def is_uri?(object)
+    is?(object, URI_RE)
+  end
+end

data/lib/one_data_accessor.rb

@@ -0,0 +1,147 @@
+require 'opennebula'
+require 'settings'
+require 'errors'
+require 'logger'
+require 'input_validator'
+
+class OneDataAccessor
+  include Errors
+  include InputValidator
+
+  STATE_DONE = '6'
+
+  attr_reader :log, :batch_size, :client, :compatibility
+
+  def initialize(compatibility, log = nil)
+    @log = log ? log : Logger.new(STDOUT)
+    @compatibility = compatibility
+
+    @batch_size = Settings.output['num_of_vms_per_file'] ? Settings.output['num_of_vms_per_file'] : 500
+    fail ArgumentError, 'Wrong number of vms per file.' unless is_number?(@batch_size)
+
+    @compatibility_vm_pool = nil
+
+    initialize_client
+  end
+
+  def initialize_client
+    secret = Settings['xml_rpc'] ? Settings.xml_rpc['secret'] : nil
+    endpoint = Settings['xml_rpc'] ? Settings.xml_rpc['endpoint'] : nil
+    fail ArgumentError, "#{endpoint} is not a valid URL." if endpoint && !is_uri?(endpoint)
+
+    @client = OpenNebula::Client.new(secret, endpoint)
+  end
+
+  def mapping(pool_class, xpath)
+    @log.debug("Generating mapping for class: #{pool_class} and xpath: '#{xpath}'.")
+    pool = pool_class.new(@client)
+    if pool.respond_to? 'info_all'
+      rc = pool.info_all
+      check_retval(rc, Errors::ResourceRetrievalError)
+    else
+      rc = pool.info
+      check_retval(rc, Errors::ResourceRetrievalError)
+    end
+
+    map = {}
+    pool.each do |item|
+      unless item['ID']
+        @log.error("Skipping a resource of the type #{pool_class} without an ID present.")
+        next
+      end
+      map[item['ID']] = item[xpath]
+    end
+
+    map
+  end
+
+  def vm(vm_id)
+    fail ArgumentError, "#{vm_id} is not a valid id." unless is_number?(vm_id)
+    @log.debug("Retrieving virtual machine with id: #{vm_id}.")
+    vm = OpenNebula::VirtualMachine.new(OpenNebula::VirtualMachine.build_xml(vm_id), @client)
+    rc = vm.info
+    check_retval(rc, Errors::ResourceRetrievalError)
+    vm
+  end
+
+  def vms(batch_number, range, groups)
+    vms = []
+    vm_pool = load_vm_pool(batch_number)
+    return nil if vm_pool.count == 0
+
+    @log.debug("Searching for vms based on range: #{range} and groups: #{groups}.")
+    vm_pool.each do |vm|
+      unless vm['ID']
+        @log.error('Skipping a record without an ID present.')
+        next
+      end
+
+      next unless want?(vm, range, groups)
+
+      vms << vm['ID'].to_i
+    end
+
+    @log.debug("Selected vms: #{vms}.")
+    vms
+  end
+
+  def want?(vm, range, groups)
+    if vm.nil?
+      @log.warn('Obtained nil vm from vm pool.')
+      return false
+    end
+    # range restriction
+    unless range.nil? || range.empty?
+      return false if range[:from] && vm['STATE'] == STATE_DONE && vm['ETIME'].to_i < range[:from].to_i
+      return false if range[:to] && vm['STIME'].to_i > range[:to].to_i
+    end
+
+    # groups restriction
+    unless groups.nil? || groups.empty?
+      return false if groups[:include] && !groups[:include].include?(vm['GNAME'])
+      return false if groups[:exclude] && groups[:exclude].include?(vm['GNAME'])
+    end
+
+    true
+  end
+
+  def load_vm_pool(batch_number)
+    fail ArgumentError, "#{batch_number} is not a valid number" unless is_number?(batch_number)
+    @log.debug("Loading vm pool with batch number: #{batch_number}.")
+    from = batch_number * @batch_size
+    to = (batch_number + 1) * @batch_size - 1
+
+    if @compatibility
+      unless @compatibility_vm_pool
+        vm_pool = OpenNebula::VirtualMachinePool.new(@client)
+        rc = vm_pool.info(OpenNebula::Pool::INFO_ALL, -1, -1, OpenNebula::VirtualMachinePool::INFO_ALL_VM)
+        check_retval(rc, Errors::ResourceRetrievalError)
+        @compatibility_vm_pool = vm_pool.to_a
+      end
+
+      return @compatibility_vm_pool[from..to] || []
+    else
+      vm_pool = OpenNebula::VirtualMachinePool.new(@client)
+      rc = vm_pool.info(OpenNebula::Pool::INFO_ALL, from, to, OpenNebula::VirtualMachinePool::INFO_ALL_VM)
+      check_retval(rc, Errors::ResourceRetrievalError)
+
+      return vm_pool
+    end
+  end
+
+  def check_retval(rc, e_klass)
+    return true unless OpenNebula.is_error?(rc)
+    case rc.errno
+    when OpenNebula::Error::EAUTHENTICATION
+      fail Errors::AuthenticationError, rc.message
+    when OpenNebula::Error::EAUTHORIZATION
+      fail Errors::UserNotAuthorizedError, rc.message
+    when OpenNebula::Error::ENO_EXISTS
+      fail Errors::ResourceNotFoundError, rc.message
+    when OpenNebula::Error::EACTION
+      fail Errors::ResourceStateError, rc.message
+    else
+      fail e_klass, rc.message
+    end
+  end
+end

data/lib/one_worker.rb

@@ -0,0 +1,181 @@
+file_dir_name = File.dirname(__FILE__)
+$LOAD_PATH.unshift(file_dir_name) unless $LOAD_PATH.include?(file_dir_name)
+
+require 'sidekiq'
+require 'one_data_accessor'
+require 'one_writer'
+require 'sidekiq_conf'
+require 'oneacct_exporter/log'
+require 'settings'
+
+class OneWorker
+  include Sidekiq::Worker
+
+  sidekiq_options retry: 5, dead: false,\
+    queue: (Settings['sidekiq'] && Settings.sidekiq['queue']) ? Settings.sidekiq['queue'].to_sym : :default
+
+  B_IN_GB = 1_073_741_824
+
+  STRING = /[[:print:]]+/
+  NUMBER = /[[:digit:]]+/
+  NON_ZERO = /[1-9][[:digit:]]*/
+  STATES = %w(started started suspended started suspended suspended completed completed suspended)
+
+  def common_data
+    common_data = {}
+    common_data['endpoint'] = Settings['endpoint']
+    common_data['site_name'] = Settings['site_name']
+    common_data['cloud_type'] = Settings['cloud_type']
+
+    common_data
+  end
+
+  def create_user_map(oda)
+    logger.debug('Creating user map.')
+    create_map(OpenNebula::UserPool, 'TEMPLATE/X509_DN', oda)
+  end
+
+  def create_image_map(oda)
+    logger.debug('Creating image map.')
+    create_map(OpenNebula::ImagePool, 'NAME', oda)
+  end
+
+  def create_map(pool_type, mapping, oda)
+    oda.mapping(pool_type, mapping)
+  rescue => e
+    msg = "Couldn't create map: #{e.message}. "\
+      'Stopping to avoid malformed records.'
+    logger.error(msg)
+    raise msg
+  end
+
+  def load_vm(vm_id, oda)
+    oda.vm(vm_id)
+  rescue => e
+    logger.error("Couldn't retrieve data for vm with id: #{vm_id}. #{e.message}. Skipping.")
+    return nil
+  end
+
+  def process_vm(vm, user_map, image_map)
+    data = common_data.clone
+
+    data['vm_uuid'] = parse(vm['ID'], STRING)
+    unless vm['STIME']
+      logger.error('Skipping a malformed record. '\
+                   "VM with id #{data['vm_uuid']} has no StartTime.")
+      return nil
+    end
+
+    data['start_time'] = parse(vm['STIME'], NUMBER)
+    start_time = data['start_time'].to_i
+    if start_time == 0
+      logger.error('Skipping a malformed record. '\
+                   "VM with id #{data['vm_uuid']} has malformed StartTime.")
+      return nil
+    end
+    data['start_time_readable'] = parse(Time.at(start_time).strftime('%F %T%:z'), STRING)
+    data['end_time'] = parse(vm['ETIME'], NON_ZERO)
+    end_time = data['end_time'].to_i
+
+    if end_time != 0 && start_time > end_time
+      logger.error('Skipping malformed record. '\
+                   "VM with id #{data['vm_uuid']} has wrong time entries.")
+      return nil
+    end
+
+    data['machine_name'] = parse(vm['DEPLOY_ID'], STRING, "one-#{data['vm_uuid']}")
+    data['user_id'] = parse(vm['UID'], STRING)
+    data['group_id'] = parse(vm['GID'], STRING)
+    data['user_name'] = parse(user_map[data['user_id']], STRING)
+    data['fqan'] = parse(vm['GNAME'], STRING, nil)
+
+    if vm['STATE']
+      data['status'] = parse(STATES[vm['STATE'].to_i], STRING)
+    else
+      data['status'] = 'NULL'
+    end
+
+    unless vm['HISTORY_RECORDS/HISTORY[1]']
+      logger.warn('Skipping malformed record. '\
+                  "VM with id #{data['vm_uuid']} has no history records.")
+      return nil
+    end
+
+    rstime = sum_rstime(vm)
+    return nil unless rstime
+
+    data['duration'] = parse(rstime.to_s, NON_ZERO)
+
+    suspend = (end_time - start_time) - data['duration'].to_i unless end_time == 0
+    data['suspend'] = parse(suspend.to_s, NUMBER)
+
+    vcpu = vm['TEMPLATE/VCPU']
+    data['cpu_count'] = parse(vcpu, NON_ZERO, '1')
+
+    net_tx = parse(vm['NET_TX'], NUMBER, 0)
+    data['network_inbound'] = (net_tx.to_i / B_IN_GB).round
+    net_rx = parse(vm['NET_RX'], NUMBER, 0)
+    data['network_outbound'] = (net_rx.to_i / B_IN_GB).round
+
+    data['memory'] = parse(vm['MEMORY'], NUMBER, '0')
+    data['image_name'] = parse(image_map[vm['TEMPLATE/DISK[1]/IMAGE_ID']], STRING)
+
+    data
+  end
+
+  def sum_rstime(vm)
+    rstime = 0
+    vm.each 'HISTORY_RECORDS/HISTORY' do |h|
+      next unless h['RSTIME'] && h['RETIME'] && h['RETIME'] != '0' && h['RSTIME'] != '0'
+      if h['RSTIME'].to_i > h['RETIME'].to_i
+        logger.warn('Skipping malformed record. '\
+                    "VM with id #{vm['ID']} has wrong CpuDuration.")
+        rstime = nil
+        break
+      end
+      rstime += h['RETIME'].to_i - h['RSTIME'].to_i
+    end
+
+    rstime
+  end
+
+  def perform(vms, output)
+    OneacctExporter::Log.setup_log_level(logger)
+
+    vms = vms.split('|')
+
+    oda = OneDataAccessor.new(logger)
+    user_map = create_user_map(oda)
+    image_map = create_image_map(oda)
+
+    data = []
+
+    vms.each do |vm_id|
+      vm = load_vm(vm_id, oda)
+      next unless vm
+
+      logger.debug("Processing vm with id: #{vm_id}.")
+      vm_data = process_vm(vm, user_map, image_map)
+      next unless vm_data
+
+      logger.debug("Adding vm with data: #{vm_data} for export.")
+      data << vm_data
+    end
+
+    write_data(data, output)
+  end
+
+  def write_data(data, output)
+    logger.debug('Creating writer...')
+    ow = OneWriter.new(data, output, logger)
+    ow.write
+  rescue => e
+    msg = "Canno't write result to #{output}: #{e.message}"
+    logger.error(msg)
+    raise msg
+  end
+
+  def parse(value, regex, substitute = 'NULL')
+    regex =~ value ? value : substitute
+  end
+end