oneacct-export 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cbb6f93cdabe253f898feaff7aa924dc5bbbffa7
-  data.tar.gz: 57bb740a6f19b3656a1c06c420d6ea3810535ba5
+  metadata.gz: 3d05de90f6c075741a02dee7cc02b1ec10633935
+  data.tar.gz: 232870fe469a478ac67710257be3768da938806d
 SHA512:
-  metadata.gz: 5eb5405b239359b3bda92ea4a895bd8eb5cb409e273918a9ea8a437579164b9877e4d8d80cab34f60dd71649b71a724e7c6f54853abea5cddf825d2aaddf51ab
-  data.tar.gz: 0863fe3000a3d01131846d3d128d62e6977e39775e887a2fe25c5694c356803de5184d7ba0d8a169cab74cd30d16c51b55c876a0b5d9cccea33256b1d30e0b09
+  metadata.gz: 930ffdb5223cd34c6b0b26929d2be5d321602dad5555cafa73d633611caf7b5874f8f991b0298d129ec869404c14a75b3b7e7752923168755285cd9746f74986
+  data.tar.gz: 4e17fba48c83dbbf1bcb6c6ffe40b6eaaadf5da82d510767cd2724a62844c3ec6525816719ce525cc37aec78866b8db1273c925624a5d691e11c0ffdba95805d

data/.travis.yml

@@ -14,3 +14,7 @@ matrix:
 branches:
   only:
     - master
+
+install: 'gem install bundler -v ">= 1.7.12" && bundle install --jobs=3 --retry=5'
+
+sudo: false

data/README.md

@@ -116,6 +116,33 @@ INHERIT_IMAGE_ATTR = "VMCATCHER_EVENT_SL_CHECKSUM_SHA512"
 INHERIT_IMAGE_ATTR = "VMCATCHER_EVENT_HV_VERSION"
 ~~~
 
+###Configure benchmark host attributes in OpenNebula
+In order to recognize and fill `BenchmarkType` and `Benchmark` APEL v0.4 fields,
+two attributes have to be set for every host in OpenNebula:
+* `BENCHMARK_TYPE` - represents benchmark's type. For example: `HEP-SPEC06`
+* `BENCHMARK_VALUES` - represents a map of virtual machine instance types and
+their measured values of said benchmark formatted as JSON. For example:
+```
+{
+  "http://schemas.fedcloud.egi.eu/occi/infrastructure/resource_tpl#atlas":312.39,
+  "http://schemas.fedcloud.egi.eu/occi/infrastructure/resource_tpl#extra_large":146.64,
+  "http://schemas.fedcloud.egi.eu/occi/infrastructure/resource_tpl#goliath":255.80,
+  "http://fedcloud.egi.eu/occi/compute/flavour/1.0#large":84.46
+}
+```
+Virtual machine instance types are in form of OCCI `scheme#term` of `resource_tpl` mixins.
+You can list your mixins via OCCI cli utility with command
+```
+occi --endpoint $ENDPOINT -a list -r resource_tpl
+```
+
+To check whether the value is correctly formatted use one of the JSON format validators and formatters, for example
+[https://jsonformatter.curiousconcept.com/](https://jsonformatter.curiousconcept.com/).
+
+Both attributes can be set both for clusters and hosts in OpenNebula with hosts'
+attributes taking precedence. If attributes are set only for cluster, all hosts
+within the cluster will be assigned these values.
+
 ###Set Rails environment variable according to your environment
 You have to set system environment variable `RAILS_ENV` to one of the
 values production, development or test. OneacctExport is not a Rails
@@ -147,9 +174,10 @@ Usage oneacct-export [options]
 
         --records-from TIME          Retrieves only records newer than TIME
         --records-to TIME            Retrieves only records older than TIME
-        --include-groups GROUP1[,GROUP2,...]
+        --records-for PERIOD         Retrieves only records within the time PERIOD
+        --include-groups [GROUP1,GROUP2,...]
                                      Retrieves only records of virtual machines which belong to the specified groups
-        --exclude-groups GROUP1[,GROUP2,...]
+        --exclude-groups [GROUP1,GROUP2,...]
                                      Retrieves only records of virtual machines which don't belong to the specified groups
         --group-file FILE            If --include-groups or --exclude-groups specified, loads groups from file FILE
     -b, --[no-]blocking              Run in a blocking mode - wait until all submitted jobs are processed
@@ -159,6 +187,19 @@ Usage oneacct-export [options]
     -v, --version                    Shows version
 ```
 
+###Package specific scripts
+When installed from packages build via [omnibus packaging for OneacctExport](https://github.com/EGI-FCTF/omnibus-oneacct-export),
+both Sidekiq and OneacctExport are automatically registered as cron jobs to run
+periodically. Cron job managing OneacctExport uses a bash script which is
+simplifying OneacctExport interface for most common use cases. After the installation,
+script can be found in `/usr/bin/oneacct-export-cron`. Script can accept command line options
+`--week|-w` (default), `--two-weeks`, `--month|-m`, `--two-months`, `--six-months`, `--year|-y` and `--all|-a`
+which sets age of retrieved records accordingly. There is also a set of files
+which when present in `/opt/oneacct-export/` directory serves as a configuration shortcut:
+* `compat.one` - turns on compatibility mode (same as OneacctExport option `--compatibility-mode`)
+* `groups.include` - contains list of groups to include (same as combination of OneacctExport options `--include-groups` and `--group-file`)
+* `groups.exclude` - contains list of groups to exclude (same as combination of OneacctExport options `--exclude-groups` and `--group-file`)
+
 ##Code Documentation
 [Code Documentation for OneacctExport by YARD](http://rubydoc.info/github/EGI-FCTF/oneacct_export/)
 
@@ -178,4 +219,3 @@ To change the log level of `oneacct-export` and `sidekiq` you have to set the en
 ```bash
 export ONEACCT_EXPORT_LOG_LEVEL=DEBUG
 ```
-

data/config/conf.yml

@@ -2,7 +2,7 @@
 defaults: &defaults
   output:
     output_dir: /var/spool/apel/outgoing/00000000 # Directory for outgoing messages
-    output_type: apel-0.2 # Format of outgoing messages. Choices are: apel-0.2, pbs-0.1, logstash-0.1
+    output_type: apel-0.4 # Format of outgoing messages. Choices are: apel-0.2, apel-0.4, pbs-0.1, logstash-0.1
     num_of_vms_per_file: 500 # Maximum number of virtual machine records per one output file
     apel: # Options for apel output format
       site_name: Undefined # Usually a short provider name, e.g. CESNET

data/lib/one_data_accessor.rb

@@ -16,6 +16,8 @@ class OneDataAccessor
   include InputValidator
 
   STATE_DONE = '6'
+  BENCHMARK_TYPE_XPATH = 'TEMPLATE/BENCHMARK_TYPE'
+  BENCHMARK_VALUES_XPATH = 'TEMPLATE/BENCHMARK_VALUES'
 
   attr_reader :log, :batch_size, :client, :compatibility
   attr_accessor :start_vm_id
@@ -197,45 +199,55 @@ class OneDataAccessor
   #
   # @return [Hash] hosts' IDs and hash with benchmark name and value
   def benchmark_map
+    map = {}
     host_pool = OpenNebula::HostPool.new(@client)
     rc = host_pool.info
     check_retval(rc, Errors::ResourceRetrievalError)
 
-    bench_map = {}
-
     host_pool.each do |host|
-      structure = {}
-      benchmark_values = nil
-      benchmark_type = nil
-
-      if (benchmark_type = host['TEMPLATE/BENCHMARK_TYPE'])
-        benchmark_values = host['TEMPLATE/BENCHMARK_VALUES'].split(/\s*\n\s*/)
-      else
-        cluster_id = host['CLUSTER_ID'].to_i
-
-        unless cluster_id == -1
-          searched_cluster = OpenNebula::Cluster.new(OpenNebula::Cluster.build_xml(cluster_id), @client)
-          rc = searched_cluster.info
-          check_retval(rc, Errors::ResourceRetrievalError)
-
-          if (benchmark_type = searched_cluster['TEMPLATE/BENCHMARK_TYPE'])
-            benchmark_values = searched_cluster['TEMPLATE/BENCHMARK_VALUES'].split(/\s*\n\s*/)
-          end
-        end
+      benchmark = benchmark_values(host)
+      if benchmark.empty?
+        cluster = cluster_for_host(host)
+        benchmark = benchmark_values(cluster) if cluster
       end
 
-      if benchmark_values
-        mixins = {}
-        benchmark_values.each do |value|
-          values = value.split(/\s+/, 2)
-          mixins[values[0]] = values[1]
-        end
-        structure = { :benchmark_type => benchmark_type, :mixins => mixins }
-      end
+      map[host['ID']] = benchmark
+    end
 
-      bench_map[host['ID']] = structure
+    map
+  end
+
+  # Returns benchmark type and values of specified entity
+  #
+  # @param [OpenNebula::PoolElement] entity
+  # @return [Hash] benchmark type and values in form of hash
+  def benchmark_values(entity)
+    benchmark_type = entity[BENCHMARK_TYPE_XPATH]
+    return {} unless benchmark_type
+
+    mixins = {}
+    benchmark_values = entity[BENCHMARK_VALUES_XPATH]
+    if benchmark_values
+      mixins = JSON.parse(benchmark_values, :max_nesting => 1)
     end
 
-    bench_map
+    { :benchmark_type => benchmark_type, :mixins => mixins }
+  end
+
+  # Returns object representing a cluster for specified host
+  #
+  # @param [OpenNebula::Host] host
+  # @return [OpenNebula::Cluster] host's cluster
+  def cluster_for_host(host)
+    cluster_id = host['CLUSTER_ID'].to_i
+
+    # host without cluster
+    return nil if cluster_id == -1
+
+    cluster = OpenNebula::Cluster.new(OpenNebula::Cluster.build_xml(cluster_id), @client)
+    rc = cluster.info
+    check_retval(rc, Errors::ResourceRetrievalError)
+
+    cluster
   end
 end

data/lib/one_worker.rb

@@ -279,16 +279,17 @@ class OneWorker
   #
   # @return [Hash] benchmark type and value or both can be nil
   def search_benchmark(vm, benchmark_map)
-    nil_benchmark = { :benchmark_type => nil, :benchmark_value => nil }
     map = benchmark_map[vm['HISTORY_RECORDS/HISTORY[last()]/HID']]
-    return nil_benchmark unless map
-    return nil_benchmark unless vm['USER_TEMPLATE/OCCI_COMPUTE_MIXINS']
+
+    return {} unless vm['USER_TEMPLATE/OCCI_COMPUTE_MIXINS']
+    return {} if map.nil? || map.empty?
 
     occi_compute_mixins = vm['USER_TEMPLATE/OCCI_COMPUTE_MIXINS'].split(/\s+/)
     occi_compute_mixins.each do |mixin|
       return { :benchmark_type => map[:benchmark_type], :benchmark_value => map[:mixins][mixin] } if map[:mixins].has_key?(mixin)
     end
-    nil_benchmark
+
+    {}
   end
 
   private

data/lib/oneacct_exporter/version.rb

@@ -1,3 +1,3 @@
 class OneacctExporter
-  VERSION = '0.4.2'
+  VERSION = '0.4.3'
 end

data/mock/one_data_accessor_cluster_01.xml

@@ -20,7 +20,7 @@
   </VNETS>
   <TEMPLATE>
     <BENCHMARK_TYPE>bench_type_cluster_1</BENCHMARK_TYPE>
-    <BENCHMARK_VALUES>mixin1 21.7</BENCHMARK_VALUES>
+    <BENCHMARK_VALUES>{"mixin1": 21.7}</BENCHMARK_VALUES>
     <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
     <RESERVED_MEM><![CDATA[8388608]]></RESERVED_MEM>
   </TEMPLATE>

data/mock/one_data_accessor_host_01.xml

@@ -49,7 +49,7 @@
   <TEMPLATE>
     <ARCH><![CDATA[x86_64]]></ARCH>
     <BENCHMARK_TYPE>bench_type_1</BENCHMARK_TYPE>
-    <BENCHMARK_VALUES>mixin1 36.9</BENCHMARK_VALUES>
+    <BENCHMARK_VALUES>{"mixin1": 36.9}</BENCHMARK_VALUES>
     <CPUSPEED><![CDATA[1200]]></CPUSPEED>
     <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
     <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>

data/mock/one_data_accessor_host_02.xml

@@ -49,8 +49,7 @@
   <TEMPLATE>
     <ARCH><![CDATA[x86_64]]></ARCH>
     <BENCHMARK_TYPE>bench_type_2</BENCHMARK_TYPE>
-    <BENCHMARK_VALUES>mixin1 788.6
-    mixin2 123.123</BENCHMARK_VALUES>
+    <BENCHMARK_VALUES>{"mixin1": 788.6, "mixin2": 123.123}</BENCHMARK_VALUES>
     <CPUSPEED><![CDATA[1200]]></CPUSPEED>
     <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
     <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>

data/spec/one_data_accessor_spec.rb

@@ -453,7 +453,7 @@ describe OneDataAccessor do
     context 'with correct data on a host with one benchmark value' do
       let(:host_filename) { 'one_data_accessor_host_01.xml' }
       let(:expected) { { '1' => { :benchmark_type => 'bench_type_1',
-                                  :mixins => { 'mixin1' => '36.9' } } } }
+                                  :mixins => { 'mixin1' => 36.9 } } } }
 
       it 'creates correct benchmark_map' do
         expect(subject.benchmark_map).to eq(expected)
@@ -463,7 +463,7 @@ describe OneDataAccessor do
     context 'with correct data on a host with two benchmark values' do
       let(:host_filename) { 'one_data_accessor_host_02.xml' }
       let(:expected) { { '2' => { :benchmark_type => 'bench_type_2',
-                                  :mixins => { 'mixin1' => '788.6', 'mixin2' => '123.123' } } } }
+                                  :mixins => { 'mixin1' => 788.6, 'mixin2' => 123.123 } } } }
 
       it 'creates correct benchmark_map' do
         expect(subject.benchmark_map).to eq(expected)
@@ -478,7 +478,7 @@ describe OneDataAccessor do
       let(:host_filename) { 'one_data_accessor_host_03.xml' }
       let(:cluster_filename) { 'one_data_accessor_cluster_01.xml' }
       let(:expected) { { '3' => { :benchmark_type => 'bench_type_cluster_1',
-                                  :mixins => { 'mixin1' => '21.7' } } } }
+                                  :mixins => { 'mixin1' => 21.7 } } } }
 
       it 'creates correct benchmark_map' do
         expect(subject.benchmark_map).to eq(expected)

data/spec/one_worker_spec.rb

@@ -602,7 +602,7 @@ describe OneWorker do
     context 'with empty benchmark_map' do
       let(:filename) { 'one_worker_vm_search_benchmark_01.xml' }
       let(:benchmark_map) { {} }
-      let(:expected) { { :benchmark_type => nil, :benchmark_value => nil } }
+      let(:expected) { {} }
 
       it 'returns array with two nil items' do
         expect(subject.search_benchmark(vm, benchmark_map)).to eq(expected)
@@ -611,7 +611,7 @@ describe OneWorker do
 
     context 'with no data for the virtual machine in benchmark_map' do
       let(:filename) { 'one_worker_vm_search_benchmark_02.xml' }
-      let(:expected) { { :benchmark_type => nil, :benchmark_value => nil } }
+      let(:expected) { {} }
 
       it 'returns array with two nil items' do
         expect(subject.search_benchmark(vm, benchmark_map)).to eq(expected)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: oneacct-export
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Michal Kimle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-28 00:00:00.000000000 Z
+date: 2016-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler