sisfc 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4db7cb93e8cfd8e52ef91b888359f61d513761a0
+  data.tar.gz: 3370237ff41037080a11ce54c4efb04510d24cab
+SHA512:
+  metadata.gz: 153d84d057e7011351998a401f44139827b72edb6051d274b8ab09bd9aaefc2a61083e26807073bd0e0a9ccfa5b53f5f4f66d654d0415cfa4743ebb2a84d0e90
+  data.tar.gz: f59a60e8c490cca293ef9836fbc990ce56663d55c681f9ecc32017ace690947a6d9ecc06728e5f1fd004762f9c87afeb050a02b790cec82079cc1ae1af244160

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sisfc.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Mauro Tortonesi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# A Simulator for IT Service in Federated Clouds (SISFC)
+
+SISFC is a simulator designed to reenact the behaviour of IT services in
+federated Cloud environments.
+
+
+## References
+
+This simulator (more precisely an earlier version of it) was used in the
+following research papers:
+
+1.  L. Foschini, M. Tortonesi, "Adaptive and Business-driven Service Placement
+    in Federated Cloud Computing Environments", in Proceedings of the 8th
+    IFIP/IEEE International Workshop on Business-driven IT Management (BDIM 2013),
+    27 May 2013, Ghent, Belgium.
+
+2.  G. Grabarnik, L. Shwartz, M. Tortonesi, "Business-Driven Optimization of
+    Component Placement for Complex Services in Federated Clouds", to appear in
+    Proceedings of the 14th IEEE/IFIP Network Operations and Management Symposium
+    (NOMS 2014) - Mini-conference track, 5-9 May 2014, Krakow, Poland.
+
+Please, consider citing some of these papers if you find this simulator useful
+for your research.
+
+
+## Installation
+
+You can install the SISFC simulator through RubyGems:
+
+    gem install sisfc
+
+While SISFC should work on MRI and Rubinius without problems, we highly
+recommend you to run it on top of JRuby. Since JRuby is our reference
+development platform, you will be very likely to have a smoother installation
+and usage experience when deploying SISFC on top of JRuby.
+
+
+## Usage
+
+To run the simulator, simply digit:
+
+    sisfc simulator.conf vm_allocation.conf
+
+where simulator.conf and vm\_allocation.conf are your simulation environment
+and vm allocation configuration files respectively.
+
+The examples directory contains a set of example configuration files, including
+an [R](http://www.r-project.org) script that models a stochastic request
+generation process. To use that script, you will need R with the VGAM and
+truncnorm packages installed.
+
+Note that the SISFC was not designed to be run directly by users, but instead
+to be integrated within automated tools that implement a continuous
+optimization framework (for instance, built on top of our [mhl metaheuristics
+library](https://github.com/mtortonesi/ruby-mhl)).

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = Dir.glob('test/**/*_test.rb').sort
+  t.verbose = true
+end

data/bin/sisfc

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+# should we actually decomment these lines?
+# require 'rubygems'
+# require 'bundler/setup'
+
+require 'awesome_print'
+
+require 'sisfc'
+require 'sisfc/evaluation'
+
+
+if __FILE__ == $0
+  unless File.exists? ARGV[0] and File.exists? ARGV[1]
+    abort("Usage: #{File.basename(__FILE__)} simulator_config_file vm_allocation_config_file")
+  end
+
+  # load simulation configuration
+  conf = SISFC::Configuration.load_from_file(ARGV[0])
+
+  # load vm allocation
+  vm_allocation = eval(File.read(ARGV[1]))
+
+  # create a simulator and launch it
+  sim = SISFC::Simulation.new(configuration: conf,
+                              evaluator: SISFC::Evaluator.new(conf))
+  res = sim.evaluate_allocation(vm_allocation)
+
+  # Print results
+  puts 'Result:'
+  ap(res, indent: 2)
+end

data/examples/generator.R

@@ -0,0 +1,45 @@
+#!/usr/bin/Rscript
+
+suppressMessages(library(VGAM))
+library(truncnorm) # for truncated normal distribution
+
+# request generation is modeled with a Pareto distribution
+# these values were chosen to generate roughly 6666.667 requests per second
+location            <- 1.2E-4
+shape               <- 5
+requests.per.second <- (shape - 1) / (shape * location)
+
+# generate 10 minutes of requests
+simulation.time <- 10 * 60
+
+# number of requests
+num.requests <- 1.2 * simulation.time * requests.per.second
+
+# latency is modeled as a gaussian distribution
+# these values were chosen to model roughly 100ms of communication latency
+mu          <-   1E-1 # mean latency is 100ms
+sigma       <- 2.5E-2 # standard deviation is 25ms
+min.latency <-   2E-2 # minimum latency is 20ms
+
+# number of data centers
+num.data.centers <- 2
+
+# request generation times
+first.request.time         <- as.POSIXct(as.Date("18/1/2013", "%d/%m/%Y"))
+request.interarrival.times <- rpareto(num.requests, location, shape)
+generation.times           <- diffinv(request.interarrival.times, xi=first.request.time)
+
+# data center ids
+data.center.ids <- sample.int(num.data.centers, length(generation.times), replace=T)
+
+# request arrival time
+latencies         <- rtruncnorm(length(generation.times), a=min.latency, b=Inf, mean=mu, sd=sigma)
+arrival.times     <- generation.times + latencies
+workflow.type.ids <- rep(1, length(generation.times))
+
+# prepare data frame and output it on the console
+df <- data.frame(Generation.Time  = generation.times,
+                 Data.Center.ID   = data.center.ids,
+                 Arrival.Time     = arrival.times,
+                 Workflow.Type.ID = workflow.type.ids)
+write.csv(df[order(df$Arrival.Time),], row.names=F)

data/examples/simulator.conf

@@ -0,0 +1,70 @@
+start_time DateTime.civil(2013,1,18,0,0,0)
+warmup_duration 10.seconds
+duration 1.minute
+
+
+data_centers \
+  1 => {
+    :maximum_vm_capacity => {
+      :medium => 50_000,
+      :large  => 50_000,
+    },
+  },
+  2 => {
+    :maximum_vm_capacity => {
+      :medium => 50_000,
+      :large  => 50_000,
+    },
+  }
+
+
+service_component_types \
+  'Web Server' => {
+    :allowed_vm_types => [ :medium ],
+    :service_time_distribution => {
+      :medium => { :distribution => :gaussian,
+                   :mean         => 0.009, # 1 request processed every 9ms
+                   :sd           => 0.001 },
+    },
+    :estimated_workload => 50,
+  },
+  'App Server' => {
+    :allowed_vm_types => [ :large ],
+    :service_time_distribution => {
+      :large  => { :distribution => :gaussian,
+                   :mean         => 0.012, # 1 request processed every 12ms
+                   :sd           => 0.002 },
+    },
+    :estimated_workload => 70,
+  }
+
+
+# workflow (or job) types descriptions
+workflow_types \
+  1 => {
+    :component_sequence => [
+      { :name => 'Web Server' }, # no need for :type => dedicated / shared
+      { :name => 'App Server' },
+    ],
+    :next_component_selection => :random,
+  }
+
+
+# input request source (the generator.R script)
+request_generation \
+  :command => '<pwd>/generator.R'
+
+
+# evaluation model
+evaluation \
+  :vm_hourly_cost => [
+    { :data_center => 1, :vm_type => :medium, :cost => 0.160 },
+    { :data_center => 1, :vm_type => :large,  :cost => 0.320 },
+    { :data_center => 2, :vm_type => :medium, :cost => 0.184 },
+    { :data_center => 2, :vm_type => :large,  :cost => 0.368 }
+  ],
+  # 500$ penalties if MTTR takes more than 50 msecs
+  :penalties => lambda {|kpis,dc_kpis| 500.0 if kpis[:mttr] > 0.050 }
+
+
+# vim: filetype=ruby

data/examples/vm_allocation.conf

@@ -0,0 +1,8 @@
+[
+  { dc_id: 1, vm_num:  7, vm_size: :medium, component_type: 'Web Server' },
+  { dc_id: 1, vm_num: 10, vm_size:  :large, component_type: 'App Server' },
+  { dc_id: 2, vm_num: 15, vm_size: :medium, component_type: 'Web Server' },
+  { dc_id: 2, vm_num: 20, vm_size:  :large, component_type: 'App Server' }
+]
+
+# vim: filetype=ruby

data/lib/sisfc/configuration.rb

@@ -0,0 +1,60 @@
+require 'sisfc/support/dsl_helper'
+
+module SISFC
+
+  module Configurable
+    dsl_accessor :start_time,
+                 :duration,
+                 :warmup_duration,
+                 :request_generation,
+                 :data_centers,
+                 :service_component_types,
+                 :evaluation,
+                 :workflow_types
+  end
+
+  class Configuration
+    include Configurable
+
+    attr_accessor :filename
+
+    def initialize(filename)
+      @filename = filename
+    end
+
+    def end_time
+      @start_time + @duration
+    end
+
+    def validate
+      # convert datetimes and integers into floats
+      @start_time      = @start_time.to_f
+      @duration        = @duration.to_f
+      @warmup_duration = @warmup_duration.to_f
+
+      # TODO: might want to restrict this substitution only to the :filename
+      # and :command keys
+      @request_generation.each do |k,v|
+        v.gsub!('<pwd>', File.expand_path(File.dirname(@filename)))
+      end
+    end
+
+    def self.load_from_file(filename)
+      # allow filename, string, and IO objects as input
+      raise ArgumentError, "File #{filename} does not exist!" unless File.exists?(filename)
+
+      # create configuration object
+      conf = Configuration.new(filename)
+
+      # take the file content and pass it to instance_eval
+      conf.instance_eval(File.new(filename, 'r').read)
+
+      # validate and finalize configuration
+      conf.validate
+
+      # return new object
+      conf
+    end
+
+  end
+end

data/lib/sisfc/data_center.rb

@@ -0,0 +1,59 @@
+module SISFC
+
+  class DataCenter
+
+    def initialize(id, opts)
+      @dcid      = id
+      @capacity  = opts[:maximum_vm_capacity]
+      @available = @capacity.dup
+      @vms       = {}
+    end
+
+    def add_vm(vm, component_name)
+      @vms[component_name] ||= []
+      raise 'Error! VM is already present!' if @vms[component_name].include? vm
+      if @available[vm.size] > 0
+        @available[vm.size] -= 1
+      else
+        raise 'Error! VM capacity exceeded!'
+      end
+      @vms[component_name] << vm
+    end
+
+    def remove_vm(vm, component_name)
+      unless @vms.has_key? component_name
+        raise "Error! Service component type #{component_name} not present in data center #{@dcid}!"
+      end
+      unless @vms[component_name].include? vm
+        raise 'Error! VM not present!'
+      end
+      @available[vm.size] += 1
+      @vms.delete(vm)
+    end
+
+    def get_random_vm(component_name)
+      unless @vms.has_key? component_name
+        raise "Error! Service component type #{component_name} not present in data center #{@dcid}!"
+      end
+      @vms[component_name].sample
+    end
+
+    def get_number_of_vms(component_name)
+      unless @vms.has_key? component_name
+        raise "Error! Service component type #{component_name} not present in data center #{@dcid}!"
+      end
+      @vms[component_name].size
+    end
+
+    def available(size)
+      @available[size]
+    end
+
+    def to_s
+      "Data center #{@dcid}, with VMs:" +
+        @vms.inject("") {|s,(k,v)| s += " (#{k}: #{v.size})" }
+    end
+
+  end
+
+end

data/lib/sisfc/evaluation.rb

@@ -0,0 +1,36 @@
+require 'sisfc/configuration'
+
+module SISFC
+  class Evaluator
+    def initialize(conf)
+      @vm_hourly_cost = conf.evaluation[:vm_hourly_cost]
+      raise ArgumentError, 'No VM hourly costs provided!' unless @vm_hourly_cost
+
+      @penalties_func = conf.evaluation[:penalties]
+    end
+
+    def evaluate_business_impact(kpis, dc_kpis, vm_allocation)
+      # evaluate VM daily costs
+      cost = vm_allocation.inject(0.0) do |s,x|
+        hc = @vm_hourly_cost.find{|i| i[:data_center] == x[:dc_id] and i[:vm_type] == x[:vm_size] }
+        unless hc
+          raise Error, "Cannot find hourly cost for data center #{x[:dc_id]} and VM size #{x[:vm_size]}!"
+        end
+        s += x[:vm_num] * hc[:cost]
+      end
+      cost *= 24.0
+      kpis[:cost] = cost
+      # puts "vm allocation cost: #{cost}"
+
+      # consider SLO violations
+      penalties = (@penalties_func.nil? ? 0.0 : @penalties_func.call(kpis, dc_kpis))
+      kpis[:penalties] = penalties
+      # puts "slo penalties cost: #{penalties}"
+      cost += penalties
+
+      # we want to minimize the cost, so we define fitness as -cost
+      fitness = -cost
+    end
+  end
+end
+

data/lib/sisfc/event.rb

@@ -0,0 +1,36 @@
+module SISFC
+
+  class Event
+
+    ET_REQUEST_ARRIVAL         = 0
+    ET_WORKFLOW_STEP_COMPLETED = 1
+    ET_VM_FREE                 = 2
+    ET_VM_SUSPEND              = 3
+    ET_VM_RESUME               = 4
+    ET_REQUEST_CLOSURE         = 5
+    ET_END_OF_SIMULATION       = 100
+
+    # let the comparable mixin provide the < and > operators for us
+    include Comparable
+
+    # should this be attr_accessor instead?
+    attr_reader :type, :data, :time, :destination
+
+    def initialize(type, data, time, destination)
+      @type        = type
+      @data        = data
+      @time        = time
+      @destination = destination
+    end
+
+    def <=> (event)
+      @time <=> event.time
+    end
+
+    def to_s
+      "Event type: #{@type}, data: #{@data}, time: #{@time}, #{@destination}"
+    end
+
+  end
+
+end

data/lib/sisfc/generator.rb

@@ -0,0 +1,80 @@
+require 'csv'
+require 'sisfc/request'
+
+
+module SISFC
+
+  class RequestGenerator
+
+    def initialize(opts)
+      if opts.has_key? :filename
+        filename = opts[:filename]
+        raise "File #{filename} does not exist!" unless File.exists?(filename)
+        @file = File.open(filename, mode='r')
+      elsif opts.has_key? :command
+        command = opts[:command]
+        @file = IO.popen(command)
+      else
+        raise ArgumentError, 'Need to provide either a filename or a command!'
+      end
+
+      # throw away the first line (containing the CSV headers)
+      @file.gets
+
+      # NOTE: so far we support only sequential integer rids
+      @next_rid = 0
+
+      setup_finalizer
+    end
+
+
+    def generate
+      # read next line from file
+      line = @file.gets
+      raise "End of input reached while reading request #{@next_rid}!" unless line
+
+      # parse data
+      tokens = line.parse_csv
+      generation_time  = tokens[0].to_f
+      data_center_id   = tokens[1].to_i
+      arrival_time     = tokens[2].to_f
+      workflow_type_id = tokens[3].to_i
+
+      # increase @next_rid
+      @next_rid += 1
+
+      # sanity check
+      if generation_time > arrival_time
+        raise "Generation time (#{generation_time}) is larger than arrival time (#{arrival_time})!"
+      end
+
+      # generate and return request
+      Request.new(@next_rid,
+                  generation_time,
+                  data_center_id,
+                  arrival_time,
+                  workflow_type_id)
+    end
+
+
+    private
+      # After object destruction, make sure that the input file is closed or
+      # the input command process is killed.
+      def setup_finalizer
+        ObjectSpace.define_finalizer(self, self.class.close_io(@file))
+      end
+
+      # Need to make this a class method, or the deallocation won't take place. See:
+      # http://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/
+      def self.close_io(file)
+        Proc.new do
+          if file.respond_to? :pid
+            Process.kill('INT', file.pid)
+          elsif
+            file.close
+          end
+        end
+      end
+  end
+
+end

data/lib/sisfc/request.rb

@@ -0,0 +1,122 @@
+module SISFC
+
+  class Request
+
+    # states
+    STATE_WORKING   = 1
+    STATE_SUSPENDED = 2
+
+    attr_accessor :rid, :generation_time, :data_center_id, :arrival_time,
+                  :workflow_type_id, :closure_time #, :status
+    attr_reader :communication_latency
+
+    def initialize(rid, generation_time, data_center_id, arrival_time, workflow_type_id)
+      @rid              = rid
+      @generation_time  = generation_time
+      @data_center_id   = data_center_id
+      @arrival_time     = arrival_time
+      @workflow_type_id = workflow_type_id
+
+      # steps count from zero
+      @step = 0
+
+      # calculate communication latency
+      @communication_latency = @arrival_time - @generation_time
+
+      # consider initial communication latency
+      @tracking_information  = [
+        {
+          :type     => :communication,
+          :at       => @generation_time,
+          :duration => @communication_latency,
+        }
+      ]
+
+      # NOTE: the format for each element of the @tracking_information array is:
+      # { :type => one of [ :queue, :work, :communication ]
+      #   :at => begin time
+      #   :duration => duration
+      #   :vm => vm (optional)
+      # }
+    end
+
+    def queuing_completed(start, duration)
+      @tracking_information << {
+        :type     => :queue,
+        :at       => start,
+        :duration => duration,
+        # :vm =>
+      }
+    end
+
+    def step_completed(start, duration)
+      @tracking_information << {
+        :type     => :work,
+        :at       => start,
+        :duration => duration,
+        # :vm =>
+      }
+      @step += 1
+    end
+
+    def finished_processing(time)
+      # consider final communication latency
+      @tracking_information << {
+        :type     => :communication,
+        :at       => time,
+        :duration => @communication_latency,
+      }
+      # save closure time
+      @closure_time = time + @communication_latency
+    end
+
+    def next_step
+      @step
+    end
+
+    def with_tracking_information(type=:all)
+      selected_ti = if type == :all
+        @tracking_information
+      else
+        @tracking_information.select{|el| el.type == type }
+      end
+
+      selected_ti.each do |ti|
+        yield ti
+      end
+    end
+
+    def total_communication_time
+      calculate_time(:communication)
+    end
+
+    def total_work_time
+      calculate_time(:work)
+    end
+
+    def total_queue_time
+      calculate_time(:queue)
+    end
+
+    def closed?
+      !@closure_time.nil?
+    end
+
+    def ttr
+      # if incident isn't closed yet, just return nil without raising an exception.
+      @closure_time.nil? ? nil : (@closure_time - @arrival_time)
+    end
+
+    def to_s
+      "rid: #{@rid}, generation_time: #{@generation_time}, data_center_id: #{@data_center_id}, arrival_time: #{@arrival_time}"
+    end
+
+    private
+
+      def calculate_time(type)
+        return 0 unless @tracking_information
+        @tracking_information.inject(0) {|sum,x| sum += ((type == :all || type == x[:type]) ? x[:duration].to_i : 0) }
+      end
+  end
+
+end

data/lib/sisfc/service_type.rb

@@ -0,0 +1,16 @@
+require 'erv'
+
+module SISFC
+  class ServiceType
+    attr_reader :level
+
+    def initialize(opts)
+      @level = opts[:level]
+      @rv    = ERV::RandomVariable.new(opts[:service_time_distribution])
+    end
+
+    def get_random_service_time
+      @rv.next
+    end
+  end
+end