oneacct-export 0.3.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e33fa6806cd2ea1a65ac02313c0a3aa3cf549095
-  data.tar.gz: 3691a9b9a6eea6c22987c3f78934c199dc7e6596
+  metadata.gz: cbb6f93cdabe253f898feaff7aa924dc5bbbffa7
+  data.tar.gz: 57bb740a6f19b3656a1c06c420d6ea3810535ba5
 SHA512:
-  metadata.gz: 0b049a0279a7cbdb37b002faa9c554596201aee6965a13789f0b234657421f4e32ce03add5708458540cfc37d80307a922837acf10028ecb60e311bdb98da60a
-  data.tar.gz: bf8ad5f4cce2931091cfcd5851780b2cbf6c42eb0f5d7e949135147387bb657653fec11cc7b8e0bfcb76fe5af69773aef608b18ece6ab873d438d9ae2978d685
+  metadata.gz: 5eb5405b239359b3bda92ea4a895bd8eb5cb409e273918a9ea8a437579164b9877e4d8d80cab34f60dd71649b71a724e7c6f54853abea5cddf825d2aaddf51ab
+  data.tar.gz: 0863fe3000a3d01131846d3d128d62e6977e39775e887a2fe25c5694c356803de5184d7ba0d8a169cab74cd30d16c51b55c876a0b5d9cccea33256b1d30e0b09

data/README.md

@@ -165,9 +165,17 @@ Usage oneacct-export [options]
 ##Continuous integration
 [Continuous integration for OneacctExport by Travis-CI](http://travis-ci.org/EGI-FCTF/oneacct_export/)
 
-## Contributing
+##Development
+###Contributing
 1. Fork it ( https://github.com/EGI-FCTF/oneacct_export/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create a new Pull Request
+
+###Debugging
+To change the log level of `oneacct-export` and `sidekiq` you have to set the environment variable **ONEACCT_EXPORT_LOG_LEVEL** to log level you need. Supported log levels are `DEBUG`, `INFO`, `WARN` and `ERROR`.
+```bash
+export ONEACCT_EXPORT_LOG_LEVEL=DEBUG
+```
+

data/bin/oneacct-export

@@ -8,6 +8,9 @@ require 'settings'
 require 'fileutils'
 require 'json'
 require 'oneacct_opts'
+require 'chronic_duration'
+
+ChronicDuration.raise_exceptions = true
 
 # parse options from command line
 options = OneacctOpts.parse(ARGV)
@@ -31,9 +34,18 @@ end
 
 OneacctExporter::Log.setup_log_level(log)
 
-range = {}
-range[:from] = options.records_from
-range[:to] = options.records_to
+begin
+  range = {}
+  range[:from] = Time.now - ChronicDuration.parse(options.records_for) if options.records_for
+
+  if options.records_from || options.records_to
+    range[:from] = options.records_from
+    range[:to] = options.records_to
+  end
+rescue ChronicDuration::DurationParseError => e
+  puts "Cannot parse a time period: #{e.message} Quitting."
+  exit
+end
 
 groups = {}
 groups[:include] = options.include_groups if options.include_groups

data/config/conf.yml

@@ -8,6 +8,7 @@ defaults: &defaults
       site_name: Undefined # Usually a short provider name, e.g. CESNET
       cloud_type: OpenNebula # CMF type, only OpenNebula is supported
       endpoint: https://occi.localhost.com:11443/ # URL of your OCCI endpoint, e.g. https://fqdn.example.com:11443/
+      # cloud_compute_service: # Name identifying cloud resource within the site. Allows multiple cloud resources within a site. i.e. a level of granularity.
     pbs: # Options for pbs output format
       realm: REALM # Owner's realm, e.g. META
       queue: cloud # Queue name

data/examples/usr/bin/oneacct-export-cron

@@ -28,17 +28,41 @@
 conf_dir="/etc/oneacct-export"
 omnibus_base_dir="/opt/oneacct-export"
 
-week_start_epoch=$((`date +%s`-604800))
-week_start_date=`date -d @$week_start_epoch +%F`
-
-if [ "x$1" = "x--all" ] || [ "x$1" = "x-all" ]; then
+if [ "x$1" = "x--all" ] || [ "x$1" = "x-a" ]; then
     # export all available accounting records
-    options="--blocking --timeout 3600"
+    options=""
+    blocking_timeout="3600"
+elif [ "x$1" = "x--two-weeks" ]; then
+    # export only records from the last two weeks
+    options="--records-for '2 weeks'"
+    blocking_timeout="1800"
+elif [ "x$1" = "x--month" ] || [ "x$1" = "x-m" ]; then
+    # export only records from the last month
+    options="--records-for month"
+    blocking_timeout="2700"
+elif [ "x$1" = "x--two-months" ]; then
+    # export only records from the last two months
+    options="--records-for '2 months'"
+    blocking_timeout="2700"
+elif [ "x$1" = "x--six-months" ]; then
+    # export only records from the last six months
+    options="--records-for '6 months'"
+    blocking_timeout="3600"
+elif [ "x$1" = "x--year" ] || [ "x$1" = "x-y" ]; then
+    # export only records from the last year
+    options="--records-for year"
+    blocking_timeout="3600"
+elif [ "x$1" = "x" ] || [ "x$1" = "x--week" ] || [ "x$1" = "x-w" ]; then
+    # export only records from the last week (default)
+    options="--records-for week"
+    blocking_timeout="1800"
 else
-    # export only records from the last 7 days
-    options="--records-from $week_start_date --blocking --timeout 1800"
+    echo "Unknown option $1."
+    exit 1
 fi
 
+options="$options --blocking --timeout $blocking_timeout"
+
 if [ -f "$conf_dir/compat.one" ]; then
     options="$options --compatibility-mode"
 fi

data/lib/data_validators/apel_data_validator.rb

@@ -42,7 +42,8 @@ module DataValidators
     # valid_data['network_outbound'] - defaults to 0
     # valid_data['memory'] - defaults to 0
     # valid_data['image_name'] - defaults to NULL
-    # valid_data['disk_size']  -defaults to NULL
+    # valid_data['disk_size'] - defaults to NULL
+    # valid_data['cloud_compute_service'] - defaults to NULL
     def validate_data(data = nil)
       unless data
         fail Errors::ValidationError, 'Skipping a malformed record. '\
@@ -61,7 +62,7 @@ module DataValidators
       valid_data['start_time'] = Time.at(start_time)
       fail_validation 'EndTime' unless number?(data['end_time'])
       end_time = data['end_time'].to_i
-      valid_data['end_time'] = end_time == 0 ? 'NULL' : Time.at(end_time)
+      valid_data['end_time'] = end_time == 0 ? DEFAULT_VALUE : Time.at(end_time)
       fail_validation 'EndTime' if end_time != 0 && valid_data['start_time'] > valid_data['end_time']
 
       valid_data['machine_name'] = default(data['machine_name'], :string, "one-#{valid_data['vm_uuid']}")
@@ -76,13 +77,13 @@ module DataValidators
         status = status.to_i
         fail_validation 'Status' unless status.to_s == data['status_code'] && status < STATES.size && status >= 0
       end
-      valid_data['status'] = status ? STATES[status] : 'NULL'
+      valid_data['status'] = status ? STATES[status] : DEFAULT_VALUE
 
       fail_validation 'HISTORY_RECORDS' if (!data['history']) || data['history'].empty?
 
       duration = sum_rstime(data['history'], valid_data['status'] == 'completed', valid_data['vm_uuid'])
       valid_data['duration'] = Time.at(duration)
-      valid_data['suspend'] = end_time == 0 ? 'NULL' : (end_time - start_time) - duration
+      valid_data['suspend'] = end_time == 0 ? DEFAULT_VALUE : (end_time - start_time) - duration
       valid_data['cpu_count'] = default(data['cpu_count'], :nzn, '1')
 
       valid_data['network_inbound'] = (default(data['network_inbound'], :number, 0).to_i / B_IN_GB).round
@@ -91,7 +92,18 @@ module DataValidators
       valid_data['memory'] = default(data['memory'], :number, '0')
       valid_data['image_name'] = default(data['image_name'], :string, DEFAULT_VALUE)
       disk_size_sum = sum_disk_size(data['disks'], valid_data['vm_uuid'])
-      valid_data['disk_size'] = disk_size_sum ? disk_size_sum : 'NULL'
+      valid_data['disk_size'] = disk_size_sum ? disk_size_sum : DEFAULT_VALUE
+
+      valid_data['cloud_compute_service'] = default(data['cloud_compute_service'], :string, DEFAULT_VALUE)
+
+      valid_data['number_of_public_ips'] = default(data['number_of_public_ips'], :number, 0)
+
+      valid_data['benchmark_type'] = default(data['benchmark_type'], :string, DEFAULT_VALUE)
+      valid_data['benchmark_value'] = default(data['benchmark_value'], :decimal, DEFAULT_VALUE)
+      if valid_data['benchmark_type'] == DEFAULT_VALUE || valid_data['benchmark_value'] == DEFAULT_VALUE
+        valid_data['benchmark_type'] = DEFAULT_VALUE
+        valid_data['benchmark_value'] = DEFAULT_VALUE
+      end
 
       valid_data
     end

data/lib/data_validators/data_validator_helper.rb

@@ -9,6 +9,7 @@ module DataValidators
     def default(value, condition_method, default_value)
       return string?(value) ? value : default_value if condition_method == :string
       return number?(value) ? value : default_value if condition_method == :number
+      return decimal?(value) ? value : default_value if condition_method == :decimal
       return non_zero_number?(value) ? value : default_value if condition_method == :nzn
     end
   end

data/lib/input_validator.rb

@@ -4,6 +4,7 @@ require 'uri'
 module InputValidator
   URI_RE = /\A#{URI.regexp}\z/
   NUMBER_RE = /\A[[:digit:]]+\z/
+  DECIMAL_RE = /\A[[:digit:]]+\.[[:digit:]]+\z/
   STRING_RE = /\A[[:print:]]+\z/
   NON_ZERO_NUMBER_RE = /\A[1-9][[:digit:]]*\z/
 
@@ -15,6 +16,10 @@ module InputValidator
     is?(object, NUMBER_RE)
   end
 
+  def decimal?(object)
+    is?(object, DECIMAL_RE) || number?(object)
+  end
+
   def uri?(object)
     is?(object, URI_RE)
   end

data/lib/one_data_accessor.rb

@@ -18,6 +18,7 @@ class OneDataAccessor
   STATE_DONE = '6'
 
   attr_reader :log, :batch_size, :client, :compatibility
+  attr_accessor :start_vm_id
 
   def initialize(compatibility, log = nil)
     @log = log ? log : Logger.new(STDOUT)
@@ -27,6 +28,7 @@ class OneDataAccessor
     fail ArgumentError, 'Wrong number of vms per file.' unless number?(@batch_size)
 
     @compatibility_vm_pool = nil
+    @start_vm_id = 0
 
     initialize_client
   end
@@ -87,15 +89,14 @@ class OneDataAccessor
 
   # Retriev IDs of specified virtual machines
   #
-  # @param [Integer] batch_number
   # @param [Hash] range date range into which virtual machine has to belong
   # @param [Hash] groups groups into one of which owner of the virtual machine has to belong
   #
   # @return [Array] array with virtual machines' IDs
-  def vms(batch_number, range, groups)
+  def vms(range, groups)
     vms = []
     # load specific batch
-    vm_pool = load_vm_pool(batch_number)
+    vm_pool = load_vm_pool
     return nil if vm_pool.count == 0
 
     @log.debug("Searching for vms based on range: #{range} and groups: #{groups}.")
@@ -145,11 +146,11 @@ class OneDataAccessor
   # Load part of virtual machine pool
   #
   # @param [Integer] batch_number
-  def load_vm_pool(batch_number)
-    fail ArgumentError, "#{batch_number} is not a valid number" unless number?(batch_number)
-    @log.debug("Loading vm pool with batch number: #{batch_number}.")
-    from = batch_number * @batch_size
-    to = (batch_number + 1) * @batch_size - 1
+  def load_vm_pool
+    @log.debug("Loading vm pool from id: #{start_vm_id}.")
+    from = @start_vm_id
+    how_many = @batch_size
+    to = from + how_many - 1
 
     # if in compatibility mode, whole virtual machine pool has to be loaded for the first time
     if @compatibility
@@ -160,12 +161,17 @@ class OneDataAccessor
         @compatibility_vm_pool = vm_pool.to_a
       end
 
-      return @compatibility_vm_pool[from..to] || []
+      pool = @compatibility_vm_pool[from..to] || []
+      @start_vm_id = pool.last.id + 1 unless pool.empty?
+
+      return pool
     else
       vm_pool = OpenNebula::VirtualMachinePool.new(@client)
-      rc = vm_pool.info(OpenNebula::Pool::INFO_ALL, from, to, OpenNebula::VirtualMachinePool::INFO_ALL_VM)
+      rc = vm_pool.info(OpenNebula::Pool::INFO_ALL, from, -how_many, OpenNebula::VirtualMachinePool::INFO_ALL_VM)
       check_retval(rc, Errors::ResourceRetrievalError)
 
+      @start_vm_id = vm_pool.entries.last.id + 1 unless vm_pool.count == 0
+
       return vm_pool
     end
   end
@@ -186,4 +192,50 @@ class OneDataAccessor
       fail e_klass, rc.message
     end
   end
+
+  # Check all hosts and gain benchmark name and value.
+  #
+  # @return [Hash] hosts' IDs and hash with benchmark name and value
+  def benchmark_map
+    host_pool = OpenNebula::HostPool.new(@client)
+    rc = host_pool.info
+    check_retval(rc, Errors::ResourceRetrievalError)
+
+    bench_map = {}
+
+    host_pool.each do |host|
+      structure = {}
+      benchmark_values = nil
+      benchmark_type = nil
+
+      if (benchmark_type = host['TEMPLATE/BENCHMARK_TYPE'])
+        benchmark_values = host['TEMPLATE/BENCHMARK_VALUES'].split(/\s*\n\s*/)
+      else
+        cluster_id = host['CLUSTER_ID'].to_i
+
+        unless cluster_id == -1
+          searched_cluster = OpenNebula::Cluster.new(OpenNebula::Cluster.build_xml(cluster_id), @client)
+          rc = searched_cluster.info
+          check_retval(rc, Errors::ResourceRetrievalError)
+
+          if (benchmark_type = searched_cluster['TEMPLATE/BENCHMARK_TYPE'])
+            benchmark_values = searched_cluster['TEMPLATE/BENCHMARK_VALUES'].split(/\s*\n\s*/)
+          end
+        end
+      end
+
+      if benchmark_values
+        mixins = {}
+        benchmark_values.each do |value|
+          values = value.split(/\s+/, 2)
+          mixins[values[0]] = values[1]
+        end
+        structure = { :benchmark_type => benchmark_type, :mixins => mixins }
+      end
+
+      bench_map[host['ID']] = structure
+    end
+
+    bench_map
+  end
 end

data/lib/one_worker.rb

@@ -6,12 +6,14 @@ require 'one_data_accessor'
 require 'one_writer'
 require 'sidekiq_conf'
 require 'oneacct_exporter/log'
+require 'oneacct_exporter/version'
 require 'settings'
 require 'data_validators/apel_data_validator'
 require 'data_validators/pbs_data_validator'
 require 'data_validators/logstash_data_validator'
 require 'output_types'
 require 'errors'
+require 'ipaddr'
 
 # Sidekiq worker class
 class OneWorker
@@ -22,23 +24,34 @@ class OneWorker
   sidekiq_options retry: 5, dead: false, \
                   queue: (Settings['sidekiq'] && Settings.sidekiq['queue']) ? Settings.sidekiq['queue'].to_sym : :default
 
+  IGNORED_NETWORKS=["10.0.0.0/8","172.16.0.0/12","192.168.0.0/16"].map {|x| IPAddr.new x}
+
+  # Prepare data that are common for all the output types
+  def common_data
+    data = {}
+    data['oneacct_export_version'] = ::OneacctExporter::VERSION
+
+    data
+  end
+
   # Prepare data that are specific for output type and common for every virtual machine
   def output_type_specific_data
     data = {}
-    if Settings.output['output_type'] == PBS_OT && Settings.output['pbs']
+    if PBS_OT.include?(Settings.output['output_type']) && Settings.output['pbs']
       data['realm'] = Settings.output.pbs['realm']
       data['pbs_queue'] = Settings.output.pbs['queue']
       data['scratch_type'] = Settings.output.pbs['scratch_type']
       data['host'] = Settings.output.pbs['host_identifier']
     end
 
-    if Settings.output['output_type'] == APEL_OT
+    if APEL_OT.include?(Settings.output['output_type'])
       data['endpoint'] = Settings.output.apel['endpoint'].chomp('/')
       data['site_name'] = Settings.output.apel['site_name']
       data['cloud_type'] = Settings.output.apel['cloud_type']
+      data['cloud_compute_service'] = Settings.output.apel['cloud_compute_service']
     end
 
-    if Settings.output['output_type'] == LOGSTASH_OT
+    if LOGSTASH_OT.include?(Settings.output['output_type'])
       data['host'] = Settings.output.logstash['host']
       data['port'] = Settings.output.logstash['port']
     end
@@ -62,6 +75,11 @@ class OneWorker
     create_map(OpenNebula::ImagePool, 'TEMPLATE/VMCATCHER_EVENT_AD_MPURI', oda)
   end
 
+  def create_cluster_map(oda)
+    logger.debug('Creating cluster map.')
+    create_map(OpenNebula::ClusterPool, 'TEMPLATE/APEL_SITE_NAME', oda)
+  end
+
   # Generic method for mapping creation
   def create_map(pool_type, mapping, oda)
     oda.mapping(pool_type, mapping)
@@ -85,8 +103,9 @@ class OneWorker
   # Obtain and parse required data from vm
   #
   # @return [Hash] required data from virtual machine
-  def process_vm(vm, user_map, image_map)
-    data = output_type_specific_data
+  def process_vm(vm, user_map, image_map, cluster_map, benchmark_map)
+    data = common_data
+    data.merge! output_type_specific_data
 
     data['vm_uuid'] = vm['ID']
     data['start_time'] = vm['STIME']
@@ -110,6 +129,14 @@ class OneWorker
     data['image_name'] ||= vm['TEMPLATE/DISK[1]/IMAGE_ID']
     data['history'] = history_records(vm)
     data['disks'] = disk_records(vm)
+    data['number_of_public_ips'] = number_of_public_ips(vm)
+
+    benchmark = search_benchmark(vm, benchmark_map)
+    data['benchmark_type'] = benchmark[:benchmark_type]
+    data['benchmark_value'] = benchmark[:benchmark_value]
+
+    site_name = cluster_map[vm['HISTORY_RECORDS/HISTORY[1]/CID']]
+    data['site_name'] = site_name if site_name
 
     data
   end
@@ -153,7 +180,23 @@ class OneWorker
     disks
   end
 
-  # Look for 'os_tpl' OCCI mixin to better identifie virtual machine's image
+  # Returns number of unique public ip addresses of vm
+  #
+  # @param [OpenNebula::VirtualMachine] vm virtual machine
+  #
+  # @return [Integer] number of unique public ip addresses represented by integer
+  def number_of_public_ips(vm)
+    all_ips = []
+    vm.each 'TEMPLATE/NIC' do |nic|
+      nic.each 'IP' do |ip|
+        all_ips << ip.text if ip_public?(ip)
+      end
+    end
+
+    all_ips.uniq.length
+  end
+
+  # Look for 'os_tpl' OCCI mixin to better identify virtual machine's image
   #
   # @param [OpenNebula::VirtualMachine] vm virtual machine
   #
@@ -185,6 +228,8 @@ class OneWorker
     oda = OneDataAccessor.new(false, logger)
     user_map = create_user_map(oda)
     image_map = create_image_map(oda)
+    cluster_map = create_cluster_map(oda)
+    benchmark_map = oda.benchmark_map
 
     data = []
 
@@ -194,11 +239,11 @@ class OneWorker
 
       begin
         logger.debug("Processing vm with id: #{vm_id}.")
-        vm_data = process_vm(vm, user_map, image_map)
+        vm_data = process_vm(vm, user_map, image_map, cluster_map, benchmark_map)
 
-        validator = DataValidators::ApelDataValidator.new(logger) if Settings.output['output_type'] == APEL_OT
-        validator = DataValidators::PbsDataValidator.new(logger) if Settings.output['output_type'] == PBS_OT
-        validator = DataValidators::LogstashDataValidator.new(logger) if Settings.output['output_type'] == LOGSTASH_OT
+        validator = DataValidators::ApelDataValidator.new(logger) if APEL_OT.include?(Settings.output['output_type'])
+        validator = DataValidators::PbsDataValidator.new(logger) if PBS_OT.include?(Settings.output['output_type'])
+        validator = DataValidators::LogstashDataValidator.new(logger) if LOGSTASH_OT.include?(Settings.output['output_type'])
 
         vm_data = validator.validate_data(vm_data) if validator
       rescue Errors::ValidationError => e
@@ -226,4 +271,38 @@ class OneWorker
     logger.error(msg)
     raise msg
   end
+
+  # Search benchmark type and value virtual machine.
+  #
+  # @param [OpenNebula::VirtualMachine] vm virtual machine
+  # @param [Hash] benchmark_map map of all hosts' benchmarks
+  #
+  # @return [Hash] benchmark type and value or both can be nil
+  def search_benchmark(vm, benchmark_map)
+    nil_benchmark = { :benchmark_type => nil, :benchmark_value => nil }
+    map = benchmark_map[vm['HISTORY_RECORDS/HISTORY[last()]/HID']]
+    return nil_benchmark unless map
+    return nil_benchmark unless vm['USER_TEMPLATE/OCCI_COMPUTE_MIXINS']
+
+    occi_compute_mixins = vm['USER_TEMPLATE/OCCI_COMPUTE_MIXINS'].split(/\s+/)
+    occi_compute_mixins.each do |mixin|
+      return { :benchmark_type => map[:benchmark_type], :benchmark_value => map[:mixins][mixin] } if map[:mixins].has_key?(mixin)
+    end
+    nil_benchmark
+  end
+
+  private
+
+  # Check if IP is public
+  #
+  # @param [String] ip address
+  #
+  # @return [Bool] true or false
+  def ip_public?(ip)
+    ip_obj = IPAddr.new(ip.text)
+    IGNORED_NETWORKS.each do |net|
+      return false if net.include? ip_obj
+    end
+    true
+  end
 end