oneacct-export 0.3.0 → 0.4.2

data/lib/one_writer.rb

@@ -3,6 +3,7 @@ require 'tempfile'
 require 'fileutils'
 require 'settings'
 require 'logger'
+require 'output_types'
 
 # Class responsible for writing data into files in specific format
 #
@@ -10,6 +11,7 @@ require 'logger'
 # @attr_reader [String] output path to the output file
 # @attr_reader [any logger] logger
 class OneWriter
+  include OutputTypes
   APEL_FILENAME_FORMAT = '%014d'
 
   attr_reader :data, :output, :log
@@ -25,7 +27,7 @@ class OneWriter
     @log = log
 
     filename = file_number.to_s
-    filename = APEL_FILENAME_FORMAT % file_number if output_type == 'apel-0.2'
+    filename = APEL_FILENAME_FORMAT % file_number if APEL_OT.include?(output_type)
     @output = "#{Settings.output['output_dir']}/#{filename}"
   end
 

data/lib/oneacct_exporter.rb

@@ -33,19 +33,17 @@ class OneacctExporter
     clean_output_dir
 
     new_file_number = 1
-    batch_number = 0
     oda = OneDataAccessor.new(@compatibility, @log)
 
     vms = []
     # load records of virtual machines in batches
-    while vms = oda.vms(batch_number, @range, @groups)
-      @log.info("Starting worker with batch number: #{batch_number}.")
+    while vms = oda.vms(@range, @groups)
       unless vms.empty?
+        @log.info("Starting worker with next batch.")
         # add a new job for every batch to the Sidekiq's queue
         OneWorker.perform_async(vms.join('|'), new_file_number)
         new_file_number += 1
       end
-      batch_number += 1
     end
 
     @log.info('No more records to read.')
@@ -56,7 +54,7 @@ class OneacctExporter
   rescue Errors::AuthenticationError, Errors::UserNotAuthorizedError,\
          Errors::ResourceNotFoundError, Errors::ResourceStateError,\
          Errors::ResourceRetrievalError => e
-    @log.error("Virtual machine retrieval for batch number #{batch_number} "\
+    @log.error("Virtual machine retrieval "\
                "failed with error: #{e.message}. Exiting.")
   end
 

data/lib/oneacct_exporter/version.rb

@@ -1,3 +1,3 @@
 class OneacctExporter
-  VERSION = '0.3.0'
+  VERSION = '0.4.2'
 end

data/lib/oneacct_opts.rb

@@ -29,6 +29,11 @@ class OneacctOpts
         options.records_to = time
       end
 
+      opts.on('--records-for PERIOD',
+              'Retrieves only records within the time PERIOD') do |period|
+        options.records_for = period
+      end
+
       opts.on('--include-groups [GROUP1,GROUP2,...]', Array,
               'Retrieves only records of virtual machines which '\
               'belong to the specified groups') do |groups|
@@ -97,6 +102,11 @@ class OneacctOpts
 
   # Make sure command line parameters are sane
   def self.check_options_restrictions(options)
+    # make sure only one time option is used
+    if (options.records_from || options.records_to) && options.records_for
+      fail ArgumentError, 'Cannot use both time period and time range options.'
+    end
+
     # make sure date range make sense
     if options.records_from && options.records_to && options.records_from >= options.records_to
       fail ArgumentError, 'Wrong time range for records retrieval.'
@@ -143,21 +153,21 @@ class OneacctOpts
   end
 
   def self.check_output_type_specific_settings
-    if Settings.output['output_type'] == APEL_OT
+    if APEL_OT.include?(Settings.output['output_type'])
       unless Settings.output['apel'] && Settings.output.apel['site_name'] &&
           Settings.output.apel['cloud_type'] && Settings.output.apel['endpoint']
         fail ArgumentError, 'Missing some mandatory parameters for APEL output type. Check your configuration file.'
       end
     end
 
-    if Settings.output['output_type'] == PBS_OT && Settings.output['pbs']
+    if PBS_OT.include?(Settings.output['output_type']) && Settings.output['pbs']
       Settings.output.pbs['realm'] ||= 'META'
       Settings.output.pbs['queue'] ||= 'cloud'
       Settings.output.pbs['scratch_type'] ||= 'local'
       Settings.output.pbs['host_identifier'] ||= 'on_localhost'
     end
 
-    if Settings.output['output_type'] == LOGSTASH_OT
+    if LOGSTASH_OT.include?(Settings.output['output_type'])
       unless Settings.output['logstash'] && Settings.output.logstash['host'] && Settings.output.logstash['port']
         fail ArgumentError, 'Missing some mandatory parameters for logstash output type. Check your configuration file.'
       end

data/lib/output_types.rb

@@ -1,5 +1,5 @@
 module OutputTypes
-  APEL_OT = 'apel-0.2'
-  PBS_OT = 'pbs-0.1'
-  LOGSTASH_OT = 'logstash-0.1'
+  APEL_OT = %w(apel-0.2 apel-0.4)
+  PBS_OT = %w(pbs-0.1)
+  LOGSTASH_OT = %w(logstash-0.1)
 end

data/lib/templates/apel-0.2.erb

@@ -25,6 +25,6 @@ Memory: <%= vm['memory']%>
 Disk: <%= vm['disk_size'].to_i != 0 ? (vm['disk_size']/1000.0).ceil : vm['disk_size']%>
 StorageRecordId: NULL
 ImageId: <%= vm['image_name']%>
-CloudType: OpenNebula
+CloudType: <%= vm['cloud_type']%>
 %%
 <% end -%>

data/lib/templates/apel-0.4.erb

@@ -0,0 +1,34 @@
+APEL-cloud-message: v0.4
+<% for vm in @data -%>
+VMUUID: <%= vm['endpoint'] %>/compute/<%= vm['vm_uuid'] %> <%= vm['start_time'].strftime('%F %T%:z') %>
+SiteName: <%= vm['site_name']%>
+CloudComputeService: <%= vm['cloud_compute_service']%>
+MachineName: <%= vm['machine_name']%>
+LocalUserId: <%= vm['user_id']%>
+LocalGroupId: <%= vm['group_id']%>
+GlobalUserName: <%= vm['user_dn']%>
+<% if vm['group_name']-%>
+FQAN: /<%= vm['group_name']%>/Role=NULL/Capability=NULL
+<% else -%>
+FQAN: NULL
+<% end -%>
+Status: <%= vm['status']%>
+StartTime: <%= vm['start_time'].to_i%>
+EndTime: <%= vm['end_time'].to_s == vm['end_time'] ? vm['end_time'] : vm['end_time'].to_i%>
+SuspendDuration: <%= vm['suspend']%>
+WallDuration: <%= vm['duration'].to_i != 0 ? vm['duration'].to_i : 'NULL'%>
+CpuDuration: <%= vm['duration'].to_i != 0 ? vm['duration'].to_i : 'NULL'%>
+CpuCount: <%= vm['cpu_count']%>
+NetworkType: NULL
+NetworkInbound: <%= vm['network_inbound']%>
+NetworkOutbound: <%= vm['network_outbound']%>
+PublicIPCount: <%= vm['number_of_public_ips']%>
+Memory: <%= vm['memory']%>
+Disk: <%= vm['disk_size'].to_i != 0 ? (vm['disk_size']/1000.0).ceil : vm['disk_size']%>
+StorageRecordId: NULL
+ImageId: <%= vm['image_name']%>
+CloudType: <%= vm['cloud_type']%> oneacct-export/<%= vm['oneacct_export_version']%>
+BenchmarkType: <%= vm['benchmark_type']%>
+Benchmark: <%= vm['benchmark_value']%>
+%%
+<% end -%>

data/mock/one_data_accessor_cluster_01.xml

@@ -0,0 +1,27 @@
+<CLUSTER>
+  <ID>1</ID>
+  <NAME>cluster-number-one</NAME>
+  <HOSTS>
+    <ID>1</ID>
+    <ID>2</ID>
+    <ID>3</ID>
+  </HOSTS>
+  <DATASTORES>
+    <ID>101</ID>
+    <ID>102</ID>
+    <ID>103</ID>
+  </DATASTORES>
+  <VNETS>
+    <ID>6</ID>
+    <ID>8</ID>
+    <ID>17</ID>
+    <ID>21</ID>
+    <ID>24</ID>
+  </VNETS>
+  <TEMPLATE>
+    <BENCHMARK_TYPE>bench_type_cluster_1</BENCHMARK_TYPE>
+    <BENCHMARK_VALUES>mixin1 21.7</BENCHMARK_VALUES>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[8388608]]></RESERVED_MEM>
+  </TEMPLATE>
+</CLUSTER>

data/mock/one_data_accessor_cluster_02.xml

@@ -0,0 +1,23 @@
+<CLUSTER>
+  <ID>2</ID>
+  <NAME>cluster-number-two</NAME>
+  <HOSTS>
+    <ID>4</ID>
+  </HOSTS>
+  <DATASTORES>
+    <ID>101</ID>
+    <ID>102</ID>
+    <ID>103</ID>
+  </DATASTORES>
+  <VNETS>
+    <ID>6</ID>
+    <ID>8</ID>
+    <ID>17</ID>
+    <ID>21</ID>
+    <ID>24</ID>
+  </VNETS>
+  <TEMPLATE>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[8388608]]></RESERVED_MEM>
+  </TEMPLATE>
+</CLUSTER>

data/mock/one_data_accessor_host_01.xml

@@ -0,0 +1,65 @@
+<HOST>
+  <ID>1</ID>
+  <NAME>whatever.in.czech.republic.cz</NAME>
+  <STATE>5</STATE>
+  <IM_MAD><![CDATA[kvm]]></IM_MAD>
+  <VM_MAD><![CDATA[kvm]]></VM_MAD>
+  <VN_MAD><![CDATA[802.1Q]]></VN_MAD>
+  <LAST_MON_TIME>1447930638</LAST_MON_TIME>
+  <CLUSTER_ID>1</CLUSTER_ID>
+  <CLUSTER>fedcloud-warg</CLUSTER>
+  <HOST_SHARE>
+    <DISK_USAGE>0</DISK_USAGE>
+    <MEM_USAGE>50331648</MEM_USAGE>
+    <CPU_USAGE>2000</CPU_USAGE>
+    <MAX_DISK>19686</MAX_DISK>
+    <MAX_MEM>57674040</MAX_MEM>
+    <MAX_CPU>3200</MAX_CPU>
+    <FREE_DISK>13945</FREE_DISK>
+    <FREE_MEM>30670728</FREE_MEM>
+    <FREE_CPU>3196</FREE_CPU>
+    <USED_DISK>4742</USED_DISK>
+    <USED_MEM>35391920</USED_MEM>
+    <USED_CPU>3</USED_CPU>
+    <RUNNING_VMS>11</RUNNING_VMS>
+    <DATASTORES>
+      <DS>
+        <FREE_MB><![CDATA[3037509]]></FREE_MB>
+        <ID><![CDATA[101]]></ID>
+        <TOTAL_MB><![CDATA[3338205]]></TOTAL_MB>
+        <USED_MB><![CDATA[122933]]></USED_MB>
+        <VMS_ALLOCATED_MB><![CDATA[122733]]></VMS_ALLOCATED_MB>
+        <VMS_REQUIRED_MB><![CDATA[130925]]></VMS_REQUIRED_MB>
+      </DS>
+    </DATASTORES>
+  </HOST_SHARE>
+  <VMS>
+    <ID>51575</ID>
+    <ID>53988</ID>
+    <ID>54035</ID>
+    <ID>54553</ID>
+    <ID>55276</ID>
+    <ID>56251</ID>
+    <ID>56288</ID>
+    <ID>56291</ID>
+    <ID>59948</ID>
+    <ID>60163</ID>
+    <ID>60173</ID>
+  </VMS>
+  <TEMPLATE>
+    <ARCH><![CDATA[x86_64]]></ARCH>
+    <BENCHMARK_TYPE>bench_type_1</BENCHMARK_TYPE>
+    <BENCHMARK_VALUES>mixin1 36.9</BENCHMARK_VALUES>
+    <CPUSPEED><![CDATA[1200]]></CPUSPEED>
+    <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
+    <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>
+    <HYPERVISOR><![CDATA[kvm]]></HYPERVISOR>
+    <MODELNAME><![CDATA[Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz]]></MODELNAME>
+    <NETRX><![CDATA[518778988428]]></NETRX>
+    <NETTX><![CDATA[761175714938]]></NETTX>
+    <PRIORITY><![CDATA[990]]></PRIORITY>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[]]></RESERVED_MEM>
+    <VERSION><![CDATA[4.10.1]]></VERSION>
+  </TEMPLATE>
+</HOST>

data/mock/one_data_accessor_host_02.xml

@@ -0,0 +1,66 @@
+<HOST>
+  <ID>2</ID>
+  <NAME>whatever.in.czech.republic.cz</NAME>
+  <STATE>5</STATE>
+  <IM_MAD><![CDATA[kvm]]></IM_MAD>
+  <VM_MAD><![CDATA[kvm]]></VM_MAD>
+  <VN_MAD><![CDATA[802.1Q]]></VN_MAD>
+  <LAST_MON_TIME>1447930638</LAST_MON_TIME>
+  <CLUSTER_ID>1</CLUSTER_ID>
+  <CLUSTER>fedcloud-warg</CLUSTER>
+  <HOST_SHARE>
+    <DISK_USAGE>0</DISK_USAGE>
+    <MEM_USAGE>50331648</MEM_USAGE>
+    <CPU_USAGE>2000</CPU_USAGE>
+    <MAX_DISK>19686</MAX_DISK>
+    <MAX_MEM>57674040</MAX_MEM>
+    <MAX_CPU>3200</MAX_CPU>
+    <FREE_DISK>13945</FREE_DISK>
+    <FREE_MEM>30670728</FREE_MEM>
+    <FREE_CPU>3196</FREE_CPU>
+    <USED_DISK>4742</USED_DISK>
+    <USED_MEM>35391920</USED_MEM>
+    <USED_CPU>3</USED_CPU>
+    <RUNNING_VMS>11</RUNNING_VMS>
+    <DATASTORES>
+      <DS>
+        <FREE_MB><![CDATA[3037509]]></FREE_MB>
+        <ID><![CDATA[101]]></ID>
+        <TOTAL_MB><![CDATA[3338205]]></TOTAL_MB>
+        <USED_MB><![CDATA[122933]]></USED_MB>
+        <VMS_ALLOCATED_MB><![CDATA[122733]]></VMS_ALLOCATED_MB>
+        <VMS_REQUIRED_MB><![CDATA[130925]]></VMS_REQUIRED_MB>
+      </DS>
+    </DATASTORES>
+  </HOST_SHARE>
+  <VMS>
+    <ID>51575</ID>
+    <ID>53988</ID>
+    <ID>54035</ID>
+    <ID>54553</ID>
+    <ID>55276</ID>
+    <ID>56251</ID>
+    <ID>56288</ID>
+    <ID>56291</ID>
+    <ID>59948</ID>
+    <ID>60163</ID>
+    <ID>60173</ID>
+  </VMS>
+  <TEMPLATE>
+    <ARCH><![CDATA[x86_64]]></ARCH>
+    <BENCHMARK_TYPE>bench_type_2</BENCHMARK_TYPE>
+    <BENCHMARK_VALUES>mixin1 788.6
+    mixin2 123.123</BENCHMARK_VALUES>
+    <CPUSPEED><![CDATA[1200]]></CPUSPEED>
+    <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
+    <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>
+    <HYPERVISOR><![CDATA[kvm]]></HYPERVISOR>
+    <MODELNAME><![CDATA[Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz]]></MODELNAME>
+    <NETRX><![CDATA[518778988428]]></NETRX>
+    <NETTX><![CDATA[761175714938]]></NETTX>
+    <PRIORITY><![CDATA[990]]></PRIORITY>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[]]></RESERVED_MEM>
+    <VERSION><![CDATA[4.10.1]]></VERSION>
+  </TEMPLATE>
+</HOST>

data/mock/one_data_accessor_host_03.xml

@@ -0,0 +1,63 @@
+<HOST>
+  <ID>3</ID>
+  <NAME>whatever.in.czech.republic.cz</NAME>
+  <STATE>5</STATE>
+  <IM_MAD><![CDATA[kvm]]></IM_MAD>
+  <VM_MAD><![CDATA[kvm]]></VM_MAD>
+  <VN_MAD><![CDATA[802.1Q]]></VN_MAD>
+  <LAST_MON_TIME>1447930638</LAST_MON_TIME>
+  <CLUSTER_ID>1</CLUSTER_ID>
+  <CLUSTER>fedcloud-warg</CLUSTER>
+  <HOST_SHARE>
+    <DISK_USAGE>0</DISK_USAGE>
+    <MEM_USAGE>50331648</MEM_USAGE>
+    <CPU_USAGE>2000</CPU_USAGE>
+    <MAX_DISK>19686</MAX_DISK>
+    <MAX_MEM>57674040</MAX_MEM>
+    <MAX_CPU>3200</MAX_CPU>
+    <FREE_DISK>13945</FREE_DISK>
+    <FREE_MEM>30670728</FREE_MEM>
+    <FREE_CPU>3196</FREE_CPU>
+    <USED_DISK>4742</USED_DISK>
+    <USED_MEM>35391920</USED_MEM>
+    <USED_CPU>3</USED_CPU>
+    <RUNNING_VMS>11</RUNNING_VMS>
+    <DATASTORES>
+      <DS>
+        <FREE_MB><![CDATA[3037509]]></FREE_MB>
+        <ID><![CDATA[101]]></ID>
+        <TOTAL_MB><![CDATA[3338205]]></TOTAL_MB>
+        <USED_MB><![CDATA[122933]]></USED_MB>
+        <VMS_ALLOCATED_MB><![CDATA[122733]]></VMS_ALLOCATED_MB>
+        <VMS_REQUIRED_MB><![CDATA[130925]]></VMS_REQUIRED_MB>
+      </DS>
+    </DATASTORES>
+  </HOST_SHARE>
+  <VMS>
+    <ID>51575</ID>
+    <ID>53988</ID>
+    <ID>54035</ID>
+    <ID>54553</ID>
+    <ID>55276</ID>
+    <ID>56251</ID>
+    <ID>56288</ID>
+    <ID>56291</ID>
+    <ID>59948</ID>
+    <ID>60163</ID>
+    <ID>60173</ID>
+  </VMS>
+  <TEMPLATE>
+    <ARCH><![CDATA[x86_64]]></ARCH>
+    <CPUSPEED><![CDATA[1200]]></CPUSPEED>
+    <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
+    <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>
+    <HYPERVISOR><![CDATA[kvm]]></HYPERVISOR>
+    <MODELNAME><![CDATA[Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz]]></MODELNAME>
+    <NETRX><![CDATA[518778988428]]></NETRX>
+    <NETTX><![CDATA[761175714938]]></NETTX>
+    <PRIORITY><![CDATA[990]]></PRIORITY>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[]]></RESERVED_MEM>
+    <VERSION><![CDATA[4.10.1]]></VERSION>
+  </TEMPLATE>
+</HOST>

data/mock/one_data_accessor_host_04.xml

@@ -0,0 +1,63 @@
+<HOST>
+  <ID>4</ID>
+  <NAME>whatever.in.czech.republic.cz</NAME>
+  <STATE>5</STATE>
+  <IM_MAD><![CDATA[kvm]]></IM_MAD>
+  <VM_MAD><![CDATA[kvm]]></VM_MAD>
+  <VN_MAD><![CDATA[802.1Q]]></VN_MAD>
+  <LAST_MON_TIME>1447930638</LAST_MON_TIME>
+  <CLUSTER_ID>2</CLUSTER_ID>
+  <CLUSTER>fedcloud-warg</CLUSTER>
+  <HOST_SHARE>
+    <DISK_USAGE>0</DISK_USAGE>
+    <MEM_USAGE>50331648</MEM_USAGE>
+    <CPU_USAGE>2000</CPU_USAGE>
+    <MAX_DISK>19686</MAX_DISK>
+    <MAX_MEM>57674040</MAX_MEM>
+    <MAX_CPU>3200</MAX_CPU>
+    <FREE_DISK>13945</FREE_DISK>
+    <FREE_MEM>30670728</FREE_MEM>
+    <FREE_CPU>3196</FREE_CPU>
+    <USED_DISK>4742</USED_DISK>
+    <USED_MEM>35391920</USED_MEM>
+    <USED_CPU>3</USED_CPU>
+    <RUNNING_VMS>11</RUNNING_VMS>
+    <DATASTORES>
+      <DS>
+        <FREE_MB><![CDATA[3037509]]></FREE_MB>
+        <ID><![CDATA[101]]></ID>
+        <TOTAL_MB><![CDATA[3338205]]></TOTAL_MB>
+        <USED_MB><![CDATA[122933]]></USED_MB>
+        <VMS_ALLOCATED_MB><![CDATA[122733]]></VMS_ALLOCATED_MB>
+        <VMS_REQUIRED_MB><![CDATA[130925]]></VMS_REQUIRED_MB>
+      </DS>
+    </DATASTORES>
+  </HOST_SHARE>
+  <VMS>
+    <ID>51575</ID>
+    <ID>53988</ID>
+    <ID>54035</ID>
+    <ID>54553</ID>
+    <ID>55276</ID>
+    <ID>56251</ID>
+    <ID>56288</ID>
+    <ID>56291</ID>
+    <ID>59948</ID>
+    <ID>60163</ID>
+    <ID>60173</ID>
+  </VMS>
+  <TEMPLATE>
+    <ARCH><![CDATA[x86_64]]></ARCH>
+    <CPUSPEED><![CDATA[1200]]></CPUSPEED>
+    <ERROR><![CDATA[Thu Nov 19 11:56:06 2015 : Error monitoring Host whatever.in.czech.republic.cz (21): -]]></ERROR>
+    <HOSTNAME><![CDATA[whatever.in.czech.republic.cz]]></HOSTNAME>
+    <HYPERVISOR><![CDATA[kvm]]></HYPERVISOR>
+    <MODELNAME><![CDATA[Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz]]></MODELNAME>
+    <NETRX><![CDATA[518778988428]]></NETRX>
+    <NETTX><![CDATA[761175714938]]></NETTX>
+    <PRIORITY><![CDATA[990]]></PRIORITY>
+    <RESERVED_CPU><![CDATA[]]></RESERVED_CPU>
+    <RESERVED_MEM><![CDATA[]]></RESERVED_MEM>
+    <VERSION><![CDATA[4.10.1]]></VERSION>
+  </TEMPLATE>
+</HOST>