oneacct-export 0.1.0

data/lib/one_writer.rb

@@ -0,0 +1,51 @@
+require 'erb'
+require 'tempfile'
+require 'fileutils'
+require 'settings'
+require 'logger'
+
+class OneWriter
+  attr_reader :data, :output, :log
+  def initialize(data, output, log = Logger.new(STDOUT))
+    fail ArgumentError, 'Data and output cannot be nil' if data.nil? || output.nil?
+
+    @template = OneWriter.template_filename(Settings.output['output_type']) if Settings['output']
+    fail ArgumentError, "No such file: #{@template}." unless File.exist?(@template)
+
+    @data = data
+    @output = output
+    @log = log
+  end
+
+  def write
+    @log.debug('Creating temporary file...')
+    tmp = Tempfile.new('oneacct_export')
+    @log.debug("Temporary file: '#{tmp.path}' created.")
+    @log.debug('Writing to temporary file...')
+    write_to_tmp(tmp, fill_template)
+    copy_to_output(tmp.path, @output)
+  ensure
+    tmp.close(true)
+  end
+
+  def write_to_tmp(tmp, data)
+    tmp.write(data)
+    tmp.flush
+  end
+
+  def copy_to_output(from, to)
+    @log.debug("Copying temporary file into '#{@output}'")
+    FileUtils.cp(from, to)
+  end
+
+  def fill_template
+    @log.debug("Reading erb template from file: '#{@template}'.")
+    erb = ERB.new(File.read(@template), nil, '-')
+    erb.filename = @template
+    erb.result(binding)
+  end
+
+  def self.template_filename(template_name)
+    "#{File.dirname(__FILE__)}/templates/#{template_name}.erb"
+  end
+end

data/lib/oneacct_exporter.rb

@@ -0,0 +1,88 @@
+require 'oneacct_exporter/version'
+require 'opennebula'
+require 'one_worker'
+require 'settings'
+require 'sidekiq/api'
+
+class OneacctExporter
+  CONVERT_FORMAT = '%014d'
+
+  attr_reader :log, :range, :groups, :blocking, :timeout, :compatibility
+
+  def initialize(options, log)
+    @log = log
+    @range = options[:range]
+    @groups = options[:groups]
+    @blocking = options[:blocking]
+    @timeout = options[:timeout]
+    @compatibility = options[:compatibility]
+  end
+
+  def export
+    @log.debug('Starting export...')
+
+    clean_output_dir
+
+    new_file_number = 1
+    batch_number = 0
+    oda = OneDataAccessor.new(@compatibility, @log)
+
+    vms = []
+    while vms = oda.vms(batch_number, @range, @groups)
+      output_file = CONVERT_FORMAT % new_file_number
+      @log.info("Starting worker with batch number: #{batch_number}.")
+      unless vms.empty?
+        OneWorker.perform_async(vms.join('|'), "#{Settings.output['output_dir']}/#{output_file}")
+        new_file_number += 1
+      end
+      batch_number += 1
+    end
+
+    @log.info('No more records to read.')
+
+    wait_for_processing if @blocking
+
+    @log.info('Exiting.')
+  rescue Errors::AuthenticationError, Errors::UserNotAuthorizedError,\
+         Errors::ResourceNotFoundError, Errors::ResourceStateError,\
+         Errors::ResourceRetrievalError => e
+    @log.error("Virtual machine retrieval for batch number #{batch_number} "\
+               "failed with error: #{e.message}. Exiting.")
+  end
+
+  def wait_for_processing
+    @log.info('Processing...')
+
+    end_time = Time.new + @timeout
+
+    until queue_empty? && all_workers_done?
+      if end_time < Time.new
+        @log.error("Processing time exceeded timeout of #{@timeout} seconds.")
+        break
+      end
+      sleep(5)
+    end
+
+    @log.info('All processing ended.')
+  end
+
+  def queue_empty?
+    queue = (Settings['sidekiq'] && Settings.sidekiq['queue']) ? Settings.sidekiq['queue'] : 'default'
+    Sidekiq::Stats.new.queues.each_pair do |queue_name, items_in_queue|
+      return items_in_queue == 0 if queue_name == queue
+    end
+
+    true
+  end
+
+  def all_workers_done?
+    Sidekiq::Workers.new.size == 0
+  end
+
+  def clean_output_dir
+    output_dir = Dir.new(Settings.output['output_dir'])
+    output_dir.entries.each do |entry|
+      File.delete("#{Settings.output['output_dir']}/#{entry}") if /[0-9]{14}/ =~ entry
+    end
+  end
+end

data/lib/oneacct_exporter/log.rb

@@ -0,0 +1,15 @@
+require 'logger'
+
+class OneacctExporter
+  module Log
+    def self.setup_log_level(logger)
+      if ENV['ONEACCT_EXPORT_LOG_LEVEL'] && Logger::Severity.const_defined?(ENV['ONEACCT_EXPORT_LOG_LEVEL'])
+        logger.level = Logger::Severity.const_get ENV['ONEACCT_EXPORT_LOG_LEVEL']
+      else
+        logger.level = Logger::INFO
+      end
+
+      logger
+    end
+  end
+end

data/lib/oneacct_exporter/version.rb

@@ -0,0 +1,3 @@
+class OneacctExporter
+  VERSION = '0.1.0'
+end

data/lib/oneacct_opts.rb

@@ -0,0 +1,131 @@
+require 'optparse'
+require 'optparse/time'
+require 'ostruct'
+require 'oneacct_exporter'
+require 'settings'
+
+class OneacctOpts
+  BLOCKING_DEFAULT = false
+  TIMEOUT_DEFAULT = 60 * 60
+  COMPATIBILITY_DEFAULT = false
+
+  def self.parse(args)
+    options = OpenStruct.new
+
+    opt_parser = OptionParser.new do |opts|
+      opts.banner = 'Usage oneacct-export [options]'
+      opts.separator ''
+
+      opts.on('--records-from TIME', Time,
+              'Retrieves only records newer than TIME') do |time|
+        options.records_from = time
+      end
+
+      opts.on('--records-to TIME', Time,
+              'Retrieves only records older than TIME') do |time|
+        options.records_to = time
+      end
+
+      opts.on('--include-groups GROUP1[,GROUP2,...]', Array,
+              'Retrieves only records of virtual machines which '\
+              'belong to the specified groups') do |groups|
+        options.include_groups = groups
+      end
+
+      opts.on('--exclude-groups GROUP1[,GROUP2,...]', Array,
+              'Retrieves only records of virtual machines which '\
+              "don't belong to the specified groups") do |groups|
+        options.exclude_groups = groups
+      end
+
+      opts.on('--group-file FILE',
+              'If --include-groups or --exclude-groups specified, '\
+              'loads groups from file FILE') do |file|
+        options.groups_file = file
+      end
+
+      opts.on('-b', '--[no-]blocking', 'Run in a blocking mode - '\
+              'wait until all submitted jobs are processed') do |blocking|
+        options.blocking = blocking
+      end
+
+      opts.on('-t', '--timeout N', Integer, 'Timeout for blocking mode in seconds. '\
+              'Default is 1 hour.') do |timeout|
+        options.timeout = timeout
+      end
+
+      opts.on('-c', '--[no-]compatibility-mode', 'Run in compatibility mode - '\
+              'supports OpenNebula 4.4.x') do |compatibility|
+        options.compatibility = compatibility
+      end
+
+      opts.on_tail('-h', '--help', 'Shows this message') do
+        puts opts
+        exit
+      end
+
+      opts.on_tail('-v', '--version', 'Shows version') do
+        puts OneacctExporter::VERSION
+        exit
+      end
+    end
+
+    opt_parser.parse!(args)
+    set_defaults(options)
+
+    check_restrictions(options)
+
+    options
+  end
+
+  def self.set_defaults(options)
+    options.blocking = BLOCKING_DEFAULT unless options.blocking
+    unless options.timeout
+      options.timeout = TIMEOUT_DEFAULT if options.blocking
+    end
+    options.compatibility = COMPATIBILITY_DEFAULT unless options.compatibility
+  end
+
+  def self.check_restrictions(options)
+    check_options_restrictions(options)
+    check_settings_restrictions
+  end
+
+  def self.check_options_restrictions(options)
+    if options.records_from && options.records_to && options.records_from >= options.records_to
+      fail ArgumentError, 'Wrong time range for records retrieval.'
+    end
+
+    if options.include_groups && options.exclude_groups
+      fail ArgumentError, 'Mixing of group options is not possible.'
+    end
+
+    unless options.include_groups || options.exclude_groups
+      if options.groups_file
+        fail ArgumentError, 'Cannot use group file without specifying group restriction type.'
+      end
+    end
+
+    if options.timeout && !options.blocking
+      fail ArgumentError, 'Cannot set timeout without a blocking mode.'
+    end
+  end
+
+  def self.check_settings_restrictions
+    unless Settings['site_name'] && Settings['cloud_type'] && Settings['endpoint'] &&
+        Settings['output'] && Settings.output['output_dir'] && Settings.output['output_type']
+      fail ArgumentError, 'Missing some mandatory parameters. Check your configuration file.'
+    end
+    Settings['endpoint'].chop! if Settings['endpoint'].end_with?('/')
+
+    if Settings['logging'] && Settings.logging['log_type'] == 'file' &&
+        !Settings.logging['log_file']
+      fail ArgumentError, 'Missing file for logging. Check your configuration file.'
+    end
+
+    template_filename = OneWriter.template_filename(Settings.output['output_type'])
+    unless File.exist?(template_filename)
+      fail ArgumentError, "Non-existing template #{Settings.output['output_type']}."
+    end
+  end
+end

data/lib/redis_conf.rb

@@ -0,0 +1,29 @@
+require 'settings'
+require 'uri'
+require 'input_validator'
+
+class RedisConf
+  extend InputValidator
+
+  def self.options
+    options = {}
+    if Settings['redis']
+      options[:namespace] = Settings.redis['namespace']
+      options[:url] = Settings.redis['url']
+    end
+
+    options[:namespace] ||= 'oneacct_export'
+    options[:url] ||= 'redis://localhost:6379'
+
+    fail ArgumentError, "#{options[:url]} is not a valid URL."\
+      unless is_uri?(options[:url])
+
+    if Settings['redis'] && Settings.redis['password']
+      fail ArgumentError, 'Redis password cannot be empty'\
+        if Settings.redis['password'].empty?
+      options[:url].insert(options[:url].index('/') + 2, ":#{Settings.redis['password']}@")
+    end
+
+    options
+  end
+end

data/lib/settings.rb

@@ -0,0 +1,13 @@
+require 'settingslogic'
+
+class Settings < Settingslogic
+  CONF_NAME = 'conf.yml'
+
+  source "#{ENV['HOME']}/.oneacct-export/#{CONF_NAME}"\
+    if File.exist?("#{ENV['HOME']}/.oneacct-export/#{CONF_NAME}")
+  source "/etc/oneacct-export/#{CONF_NAME}"\
+    if File.exist?("/etc/oneacct-export/#{CONF_NAME}")
+  source "#{File.dirname(__FILE__)}/../config/#{CONF_NAME}"
+
+  namespace ENV['RAILS_ENV'] ? ENV['RAILS_ENV'] : 'production'
+end

data/lib/sidekiq_conf.rb

@@ -0,0 +1,11 @@
+require 'sidekiq'
+require 'redis_conf'
+
+Sidekiq.configure_client do |config|
+  options = RedisConf.options
+  options[:size] = 1
+  config.redis = options
+end
+Sidekiq.configure_server do |config|
+  config.redis = RedisConf.options
+end

data/lib/templates/apel-0.2.erb

@@ -0,0 +1,30 @@
+APEL-cloud-message: v0.2
+<% for vm in @data -%>
+VMUUID: <%= vm['endpoint'] %>/compute/<%= vm['vm_uuid'] %> <%= vm['start_time_readable']%>
+SiteName: <%= vm['site_name']%>
+MachineName: <%= vm['machine_name']%>
+LocalUserId: <%= vm['user_id']%>
+LocalGroupId: <%= vm['group_id']%>
+GlobalUserName: <%= vm['user_name']%>
+<% if vm['fqan']-%>
+FQAN: /<%= vm['fqan']%>/Role=NULL/Capability=NULL
+<% else -%>
+FQAN: NULL
+<% end -%>
+Status: <%= vm['status']%>
+StartTime: <%= vm['start_time']%>
+EndTime: <%= vm['end_time']%>
+SuspendDuration: <%= vm['suspend']%>
+WallDuration: <%= vm['duration']%>
+CpuDuration: <%= vm['duration']%>
+CpuCount: <%= vm['cpu_count']%>
+NetworkType: NULL
+NetworkInbound: <%= vm['network_inbound']%>
+NetworkOutbound: <%= vm['network_outbound']%>
+Memory: <%= vm['memory']%>
+Disk: NULL
+StorageRecordId: NULL
+ImageId: <%= vm['image_name']%>
+CloudType: OpenNebula
+%%
+<% end -%>

data/mock/one_worker_DEPLOY_ID_missing.xml

@@ -0,0 +1,136 @@
+<VM>
+  <ID>36551</ID>
+  <UID>120</UID>
+  <GID>0</GID>
+  <UNAME>uname</UNAME>
+  <GNAME>gname</GNAME>
+  <NAME>one-36551</NAME>
+  <PERMISSIONS>
+    <OWNER_U>1</OWNER_U>
+    <OWNER_M>1</OWNER_M>
+    <OWNER_A>0</OWNER_A>
+    <GROUP_U>0</GROUP_U>
+    <GROUP_M>0</GROUP_M>
+    <GROUP_A>0</GROUP_A>
+    <OTHER_U>0</OTHER_U>
+    <OTHER_M>0</OTHER_M>
+    <OTHER_A>0</OTHER_A>
+  </PERMISSIONS>
+  <LAST_POLL>1383741679</LAST_POLL>
+  <STATE>6</STATE>
+  <LCM_STATE>0</LCM_STATE>
+  <RESCHED>0</RESCHED>
+  <STIME>1383741160</STIME>
+  <ETIME>1383742270</ETIME>
+  <MEMORY>1736960</MEMORY>
+  <CPU>0</CPU>
+  <NET_TX>0</NET_TX>
+  <NET_RX>0</NET_RX>
+  <TEMPLATE>
+    <CONTEXT>
+      <DISK_ID><![CDATA[1]]></DISK_ID>
+      <FILES><![CDATA[https://somewhere.com/init.sh]]></FILES>
+      <HOSTNAME><![CDATA[DebianVM]]></HOSTNAME>
+      <PUBLIC_IP><![CDATA[123.123.5.5]]></PUBLIC_IP>
+      <SSH_KEY><![CDATA[ssh-rsa AAAAB3NzaC1yc2EAAAADAQABasdgsdfhfgjhxhcHPNFthrHT5/+9lfrQCorJy5YjMJEGBC22dfgjghkDjSDFgjdhgKd54645dfJKd39LdlyzlEaCnjGUeD4C6tZdAISLSDfghSDFgSJ54sSFGJ5RTHARJIRJcX8QRVH46Zq02AeTgrty567ss34tqaa8Pt user@machine1.somewhere.com]]></SSH_KEY>
+      <TARGET><![CDATA[xvdb]]></TARGET>
+    </CONTEXT>
+    <CPU><![CDATA[0.5]]></CPU>
+    <DISK>
+      <BUS><![CDATA[ide]]></BUS>
+      <CLONE><![CDATA[YES]]></CLONE>
+      <CLUSTER_ID><![CDATA[100]]></CLUSTER_ID>
+      <DATASTORE><![CDATA[default]]></DATASTORE>
+      <DATASTORE_ID><![CDATA[1]]></DATASTORE_ID>
+      <DEV_PREFIX><![CDATA[xvd]]></DEV_PREFIX>
+      <DISK_ID><![CDATA[0]]></DISK_ID>
+      <DRIVER><![CDATA[tap2:tapdisk:aio:]]></DRIVER>
+      <IMAGE><![CDATA[debian6]]></IMAGE>
+      <IMAGE_ID><![CDATA[31]]></IMAGE_ID>
+      <READONLY><![CDATA[NO]]></READONLY>
+      <SAVE><![CDATA[NO]]></SAVE>
+      <SOURCE><![CDATA[/opt/opennebula/var/datastores/1/bc8db254875412f417ac745b7d47820]]></SOURCE>
+      <TARGET><![CDATA[xvda]]></TARGET>
+      <TM_MAD><![CDATA[ssh]]></TM_MAD>
+      <TYPE><![CDATA[FILE]]></TYPE>
+    </DISK>
+    <ERROR>
+      <MESSAGE><![CDATA[Error executing image transfer script: Path https://somewhere.com/init.sh is not allowed!]]></MESSAGE>
+      <TIMESTAMP><![CDATA[Wed Nov  6 13:34:19 2013]]></TIMESTAMP>
+    </ERROR>
+    <FEATURES>
+      <ACPI><![CDATA[no]]></ACPI>
+    </FEATURES>
+    <MEMORY>1736960</MEMORY>
+    <NAME><![CDATA[one-36551]]></NAME>
+    <NIC>
+      <BRIDGE><![CDATA[xenbr0]]></BRIDGE>
+      <CLUSTER_ID><![CDATA[100]]></CLUSTER_ID>
+      <IP><![CDATA[123.123.5.5]]></IP>
+      <MAC><![CDATA[01:23:45:67:89:ab]]></MAC>
+      <NETWORK><![CDATA[public]]></NETWORK>
+      <NETWORK_ID><![CDATA[4]]></NETWORK_ID>
+      <VLAN><![CDATA[NO]]></VLAN>
+      <WHITE_PORTS_TCP><![CDATA[22]]></WHITE_PORTS_TCP>
+      <WHITE_PORTS_UDP><![CDATA[67,68]]></WHITE_PORTS_UDP>
+    </NIC>
+    <OS>
+      <BOOTLOADER><![CDATA[pygrub]]></BOOTLOADER>
+    </OS>
+    <RAW>
+      <TYPE><![CDATA[xen]]></TYPE>
+    </RAW>
+    <TEMPLATE_ID><![CDATA[66]]></TEMPLATE_ID>
+    <VCPU><![CDATA[1]]></VCPU>
+    <VMID><![CDATA[36551]]></VMID>
+  </TEMPLATE>
+  <USER_TEMPLATE>
+    <SCHED_REQUIREMENTS><![CDATA[CLUSTER_ID = 100]]></SCHED_REQUIREMENTS>
+  </USER_TEMPLATE>
+  <HISTORY_RECORDS>
+    <HISTORY>
+      <OID>36551</OID>
+      <SEQ>0</SEQ>
+      <HOSTNAME>supermachine1.somewhere.com</HOSTNAME>
+      <HID>11</HID>
+      <CID>100</CID>
+      <STIME>1383741169</STIME>
+      <ETIME>1383741259</ETIME>
+      <VMMMAD>vmm_xen</VMMMAD>
+      <VNMMAD>fw</VNMMAD>
+      <TMMAD>ssh</TMMAD>
+      <DS_LOCATION>/opt/opennebula/var/datastores</DS_LOCATION>
+      <DS_ID>0</DS_ID>
+      <PSTIME>1383741169</PSTIME>
+      <PETIME>1383741259</PETIME>
+      <RSTIME>0</RSTIME>
+      <RETIME>0</RETIME>
+      <ESTIME>0</ESTIME>
+      <EETIME>0</EETIME>
+      <REASON>1</REASON>
+      <ACTION>0</ACTION>
+    </HISTORY>
+    <HISTORY>
+      <OID>36551</OID>
+      <SEQ>1</SEQ>
+      <HOSTNAME>supermachine1.somewhere.com</HOSTNAME>
+      <HID>11</HID>
+      <CID>100</CID>
+      <STIME>1383741589</STIME>
+      <ETIME>1383742270</ETIME>
+      <VMMMAD>vmm_xen</VMMMAD>
+      <VNMMAD>fw</VNMMAD>
+      <TMMAD>ssh</TMMAD>
+      <DS_LOCATION>/opt/opennebula/var/datastores</DS_LOCATION>
+      <DS_ID>0</DS_ID>
+      <PSTIME>1383741589</PSTIME>
+      <PETIME>1383741674</PETIME>
+      <RSTIME>1383741674</RSTIME>
+      <RETIME>1383742270</RETIME>
+      <ESTIME>1383742270</ESTIME>
+      <EETIME>1383742270</EETIME>
+      <REASON>0</REASON>
+      <ACTION>0</ACTION>
+    </HISTORY>
+  </HISTORY_RECORDS>
+</VM>