cloud_financial_officer 0.1.0

data/lib/cfo/record_grouper.rb

@@ -0,0 +1,102 @@
+module CFO
+  # Encapsulates sevral algorithms for for changing one set of BillingRecords
+  # into another.
+  module RecordGrouper
+    include CloudCostTracker::Billing
+
+    # Maps the grouping Strings to BillingRecord attributes.
+    # Any strings not found as keys in this Hash are interpreted as
+    # BillingCode keys.
+    ATTR_4_GROUPING = {
+      'account' => :account, 'service' => :service, 'provider' => :provider,
+      'type' => :resource_type, 'resource_id' => :resource_id,
+      'billing_type' => :billing_type,
+    }
+
+    # Defines the value put assigned to composite record's attibute when all
+    # its commponent records don't share the same value.
+    MULTIPLE_VALUES_VALUE = '*'
+
+    # Returns a set of "composite" BillingRecords based on an Array of groupings.
+    # If a grouping is one of ['account', 'provider', 'service', 'resource_id',
+    # 'resource_type', 'billing_type'], it is evaluated as a Record's attribute.
+    # Otherwise, its treated as a BillingCode key, and evaluated to it's value.
+    # @param [Array<BillingRecord>] input_records the records to group together
+    # @param [Array<String>] groupings the paramters to group records by
+    # @return [Array<BillingRecord>] the composite BillingRecords
+    def self.by(groupings, input_records)
+      # index the results by a "composite key", made by evaluating the groupings
+      results = Hash.new
+      input_records.each do |record|
+        # create the composite key for this record
+        comp_key = composite_key_for(record, groupings)
+        # if there is already composite record for this key,
+        if comp_record = results[comp_key]
+          # merge this record's data into the composite record
+          results[comp_key] = combine_records(comp_record, record)
+        else  # otherwise, create a new composite record
+          # index the record into the results with the composite key
+          results[comp_key] = record
+        end
+      end
+      # Now output the results in composite key order
+      results.keys.sort.map {|comp_key| results[comp_key]}
+    end
+
+    private
+
+    # Generates a composite key for a given record based on a list of groupings
+    def self.composite_key_for(record, groupings)
+      groupings.inject("") do |comp_key, grouping|
+        if ATTR_4_GROUPING[grouping] # this grouping represents an attribute
+          comp_key += record.send(ATTR_4_GROUPING[grouping])
+        else  # this grouping is a BillingCode key -- search for the BillingCode
+          new_partial_key = '*UNDEFINED_OR_EMPTY*'
+          if code = record.billing_codes.find {|c| c.key == grouping}
+            new_partial_key = code.value if (code.value && (code.value != ''))
+          end
+          comp_key += new_partial_key
+        end
+      end
+    end
+
+    # Merges some information from record over a composite record comp_record,
+    # based on the provided groupings.
+    def self.combine_records(comp_record, record)
+      comp_record.total_cost += record.total_cost
+      comp_record.start_time = [comp_record.start_time, record.start_time].min
+      comp_record.stop_time  = [comp_record.stop_time,  record.stop_time].max
+      # recalculate the hourly rate for the composite record
+      duration = comp_record.stop_time - comp_record.start_time
+      comp_record.cost_per_hour = duration != 0 ?
+        comp_record.total_cost * SECONDS_PER_HOUR / duration : 0
+      comp_record.billing_codes =
+        (comp_record.billing_codes + record.billing_codes).uniq
+      ATTR_4_GROUPING.values.each do |attribute|
+        existing_value = comp_record.send(attribute)
+        new_record_value = record.send(attribute)
+        if existing_value != new_record_value
+          comp_record.send("#{attribute.to_s}=", MULTIPLE_VALUES_VALUE)
+        end
+      end
+      comp_record
+    end
+
+    # Creates a new composite BillingRecord with only the attributes matching the
+    # grouping fields.
+    def self.create_composite_record(record, groupings)
+      record_attribs = groupings.inject({}) do |attrs, grouping|
+        if ATTR_4_GROUPING[grouping] # this grouping String represents an attribute
+          attrs[ATTR_4_GROUPING[grouping]] = record.send(ATTR_4_GROUPING[grouping])
+        end
+        attrs
+      end
+      BillingRecord.new(record_attribs.merge({
+        :total_cost => record.total_cost, :cost_per_hour => record.cost_per_hour,
+        :start_time => record.start_time, :stop_time => record.stop_time,
+        :billing_codes => record.billing_codes
+      }))
+    end
+
+  end
+end

data/lib/cfo/resources/account.rb

@@ -0,0 +1,53 @@
+module CFO
+  # Represents an account for a specific service with a cloud provider.
+  class Account
+    # A human-readable ID for the account - best to make this URL-safe.
+    attr_accessor :name
+
+    # The (Fog) name of the cloud computing service provider.
+    attr_accessor :provider
+
+    # The (Fog) name of the cloud computing service.
+    attr_accessor :service
+
+    # An Array of String pairs representing all the BillingCodes associated
+    # with all this Acccount's BillingRecords.
+    attr_accessor :billing_codes
+
+    # Creates a object representing a REST Resource for an account.
+    # @param attributes [Hash] - :name, :provider, :service, or :billing_codes
+    def initialize(attributes = {})
+      attributes.each {|k,v| self.send("#{k}=",v) if self.respond_to?("#{k}=")}
+    end
+
+    # @return [Hash] a RESTful Hash representation of this object.
+    # @param [Booolean] verbose whether to include biling codes and resources.
+    def to_hash(verbose = true)
+      extra_info = {
+        'resources' => CloudResource.hash_list(:account => @name),
+        'billing_codes' => @billing_codes,
+      }
+      {
+        'id'            => @name,
+        'service'       => @service,
+        'provider'      => @provider,
+        'url'           => uri,
+        'report_url'    => report_uri,
+      }.merge(verbose ? extra_info : {})
+    end
+
+    # @return [String] a URI path to this object's JSON representation
+    def uri
+      "#{CFO.root}/accounts/#{URI.escape(@name)}"
+    end
+
+    # @return [String] a URI path to this object's cost report.
+    def report_uri
+      "#{CFO.root}/report/for_account/#{URI.escape(@name)}"
+    end
+
+    # @return [String] a RESTful JSON representation of this object.
+    def to_json ; to_hash.to_json end
+
+  end
+end

data/lib/cfo/resources/cloud_resource.rb

@@ -0,0 +1,74 @@
+module CFO
+  # Represents a billable asset on a cloud computing service.
+  class CloudResource
+    include CloudCostTracker::Billing
+
+    # A CFO::Account object.
+    attr_accessor :account
+
+    # The resource type, unique this account.
+    attr_accessor :resource_type
+
+    # The resource ID, unique to this account and resource type.
+    attr_accessor :resource_id
+
+    # An Array of String pairs representing all the BillingCodes associated
+    # with all this Resource's BillingRecords.
+    attr_accessor :billing_codes
+
+    # Creates a object representing a REST Resource for a billable resource.
+    # @param attributes [Hash] - :account, :resource_type,
+    #   :resource_id, or :billing_codes
+    def initialize(attributes = {})
+      attributes.each {|k,v| self.send("#{k}=",v) if self.respond_to?("#{k}=")}
+    end
+
+    # @return [Hash] a RESTful Hash representation of this object.
+    # @param [Booolean] verbose whether to include a full representation.
+    def to_hash(verbose = true)
+      extra_data = {
+        'account' => {
+          'id'          => @account.name,
+          'service'     => @account.service,
+          'provider'    => @account.provider,
+          'url'         => @account.uri,
+        },
+        'report_url'    => report_uri,
+        'billing_codes' => @billing_codes,
+      }
+      { # Return the resource Hash, perhaps with the account merged in
+        'resource_id'   => @resource_id,
+        'resource_type' => @resource_type,
+        'url'           => uri,
+      }.merge(verbose ? extra_data : {})
+    end
+
+    # @return [String] a URI path to this object's JSON representation
+    def uri
+      "#{CFO.root}/accounts/#{URI.escape(@account.name)}/"+
+        "#{URI.escape(@resource_type)}/#{URI.escape(@resource_id)}"
+    end
+
+    # @return [String] a URI path to this object's cost report.
+    def report_uri
+      "#{CFO.root}/report/for_account/#{URI.escape(@account.name)}/for_type/"+
+        "#{URI.escape(@resource_type)}/for_resource/#{URI.escape(@resource_id)}"
+    end
+
+    # @return [String] a RESTful JSON representation of this object.
+    def to_json ; to_hash.to_json end
+
+    # @return [Array<Resource>] a 'links-only' list of matcing Resources,
+    #   loaded from the database.
+    def self.hash_list(conditions = {})
+      BillingRecord.select('*').group(:resource_id).where(conditions).
+      collect do |rec|
+        CloudResource.new(
+          rec.attributes.merge('account' =>
+            CFO::Account.new(rec.attributes.merge(:name => rec.account)))
+        ).to_hash(false)  # Don't include the full account info
+      end
+    end
+
+  end
+end

data/lib/cfo/resources/report.rb

@@ -0,0 +1,147 @@
+module CFO
+  # Represents a JSON report, based on an Array of (Hash) constraints.
+  class Report
+    include CloudCostTracker::Billing
+
+    # Creates a object representing a REST Resource for an account.
+    # @param [Array<Hash>] constraint_array a set of Hash constraints on the
+    #   BillingRecords to be retrieved.
+    #   Each hash key is a type of constraint on the type of BillingRecord
+    #   information that is returned. The value is an array of Strings, which
+    #   are generally just text to search for in some BillingRecord column, but
+    #   are interpreted differently depending on the type of constraint.
+    # @param [Array<String>] groupings a set of Strings for grouping the results.
+    #   Strings that match BillingRecord attributes will group all records that
+    #   share the same value for that attribute into one record.
+    #   Other Strings will can match BillingCode keys, for grouping by Code.
+    # @param [Hash] options optional additional parameters:
+    #  - :logger - a Ruby Logger-compatible object
+    def initialize(constraint_array = [], groupings = [], options = {})
+      @constraints = constraint_array
+      @groupings = groupings
+      @log = options[:logger] || FogTracker.default_logger(nil)
+    end
+
+    # @return [Hash] a Hash representation of the desired cost report.
+    def to_hash
+      update_records            # Update @records from the database
+      update_summary_attributes # Update summary attributes
+      {
+        'start_time'      => @start_time,
+        'stop_time'       => @stop_time,
+        'total_cost'      => @total_cost,
+        'cost_per_hour'   => @hourly_rate.to_f,
+        'billing_codes'   => @bill_codes,
+        'billing_records' => @records.map do |r|
+                                r.to_hash.merge(
+                                  :cost_per_hour  => r.cost_per_hour.to_f,
+                                  :total_cost     => r.total_cost.to_f,
+                                )
+                              end,
+      }
+    end
+
+    # @return [String] a CSV representation of the desired cost report.
+    def to_csv
+      update_records            # Update @records from the database
+      code_count = @records.inject({}) do |count, record|
+        record.billing_codes.each do |code|
+          count[code.key] = count[code.key] ? count[code.key] + 1 : 1
+        end
+        count
+      end
+      sorted_codes = code_count.keys.sort {|a,b| code_count[b] <=> code_count[a]}
+      headers = {
+        :start_time => 'Start Time',
+        :stop_time => 'Stop Time',
+        :account => 'Account Name',
+        :provider => 'Provder',
+        :service => 'Service',
+        :billing_type => 'Billing Type',
+        :total_cost => 'Total Cost',
+        :cost_per_hour => 'Hourly Rate',
+        :resource_id => 'Resource ID',
+        :resource_type => 'Resource Type',
+      }
+      @log.info "Using headers: #{headers.values + sorted_codes}"
+      csv = ((headers.values + sorted_codes).map{|h| "\"#{h}\""}).join(',') +"\n"
+      @records.each do |record|
+        record_data = headers.keys.map {|h| record.send h}
+        record_data += sorted_codes.map do |code|
+          billing_codes = record.billing_codes.select {|c| c.key == code}
+          case billing_codes.count
+          when 0 then ''
+          when 1 then billing_codes.first.value
+          else CFO::RecordGrouper::MULTIPLE_VALUES_VALUE
+          end
+        end
+        csv += (record_data.map{|value| "\"#{value}\""}).join(',') +"\n"
+      end
+      csv
+    end
+
+    # @return [String] a JSON report on the desired costs.
+    def to_json ; to_hash.to_json end
+
+    # Sets a constraint based on resource IDs.
+    # @param [Array<String>] resource_ids the values to match in the records.
+    def for_resource(resource_ids) ;  {:resource_id => resource_ids} end
+
+    # Sets a constraint based on resource type.
+    # @param [Array<String>] resource_types the values to match in the records.
+    def for_type(resource_types) ;    {:resource_type => resource_types} end
+
+    # Sets a constraint based on account name.
+    # @param [Array<String>] account_names the values to match in the records.
+    def for_account(account_names) ;  {:account => account_names} end
+
+    # Sets a constraint based on provider name.
+    # @param [Array<String>] providers the values to match in the records.
+    def for_provider(providers) ;     {:provider => providers} end
+
+    # Sets a constraint based on cloud computing service name.
+    # @param [Array<String>] services the values to match in the records.
+    def for_service(services) ;       {:service => services} end
+
+    # Sets a constraint based on billing type.
+    # @param [Array<String>] billing_types the values to match in the records
+    def for_billing_type(billing_types) ; {:billing_type => billing_types} end
+
+    private
+
+    # Updates @records from the database.
+    def update_records
+      conditions = Hash.new
+      @constraints.each do |constraint_hash|
+        constraint_hash.each do |constraint_key, values|
+          if self.respond_to? constraint_key
+            conditions.merge!(self.send(constraint_key, values))
+          end
+        end
+      end
+      # Query for BillingRecords, but freeze the resulting records
+      @records = BillingRecord.find(
+        :all, :conditions => conditions, :readonly => true
+      )
+      @log.debug "Got #{@records.count} records for constraints #{conditions}."
+      @records = RecordFilter.by_time(@records, @constraints)
+      @records = RecordFilter.by_code(@records, @constraints)
+      @records = RecordGrouper.by(@groupings, @records) if not @groupings.empty?
+    end
+
+    # Updates summary attributes for time range, cost and billing codes.
+    def update_summary_attributes
+      @start_time   = (@records.map{|r| r.start_time}).sort.first
+      @stop_time    = (@records.map{|r| r.stop_time}).sort.last
+      @total_cost   = @records.inject(0.0) {|total, r| total += r.total_cost}
+      duration      = (@start_time && @stop_time) ? (
+                        Time.parse(@stop_time.to_s) - Time.parse(@start_time.to_s)
+                      ) : 0
+      @hourly_rate  = duration != 0 ? @total_cost * SECONDS_PER_HOUR / duration : 0
+      @bill_codes   = (@records.map {|r| r.billing_codes}).flatten.uniq.map do |c|
+                      [c.key, c.value]
+                    end
+    end
+
+  end
+end

data/lib/cfo/service.rb

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'cloud_financial_officer')
+require 'sinatra/base'
+CFO.force_timezone  # This does nothing by default - modify it if you need it
+
+module CFO
+  # A RESTful web service for reporting on a CloudCostTracker database.
+  class Service < Sinatra::Base
+    include ::CloudCostTracker::Billing
+
+    # Load account information
+    configure do ; set :accounts, CFO.read_accounts end
+
+    configure :development do
+      require 'sinatra/reloader'
+      register Sinatra::Reloader
+    end
+    configure :production, :development do
+      enable :logging
+      ActiveRecord::Base.logger ||= FogTracker.default_logger(STDOUT)
+      #ActiveRecord::Base.logger.level = ::Logger::DEBUG
+    end
+    before do
+      if not ActiveRecord::Base.connected? # Connect to the database as needed
+        logger.info "Using database #{(CFO.connect_to_database)[:database]}..."
+      end
+      logger.info "Got params #{params}"
+      if params['content-type'] && (params['content-type'] =~ %r{(text/)?csv})
+        content_type 'text/csv'         # serve only JSON by default
+      else
+        content_type 'application/json' # serve only JSON by default
+      end
+    end
+
+
+    ######## ACCOUNTS ########
+
+    # Returns a list of accounts from / or /accounts.
+    get %r{^#{CFO.root}(/accounts)?/?$} do
+      (settings.accounts.map{|account| account.to_hash(false)}).to_json
+    end
+
+    # Returns an individial account.
+    get "#{CFO.root}/accounts/:account_name" do
+      if account = settings.accounts.find {|a| a.name == params[:account_name]}
+        account.billing_codes = BillingCode.joins(:billing_records).where(
+          :billing_records => {:account => params[:account_name]}
+        ).uniq.map {|c| [c.key, c.value]}
+        account.to_json
+      else
+        [404, {'error' => "Account #{params[:account_name]} not found"}.to_json]
+      end
+    end
+
+    ######## RESOURCES ########
+
+    # Returns a list of CloudResources.
+    get %r{^#{CFO.root}/resources/?$} do
+      CFO::CloudResource.hash_list.to_json
+    end
+
+    # Returns an individial CloudResource.
+    get "#{CFO.root}/accounts/:account_name/:type/:resource_id" do
+      all_records = BillingRecord.where(
+        :account        => params[:account_name],
+        :resource_type  => params[:type],
+        :resource_id    => params[:resource_id],
+      )
+      if record = all_records.first
+        codes = BillingCode.joins(:billing_records).where(
+          :billing_records => {:resource_id => params[:resource_id]}
+        )
+        CFO::CloudResource.new(
+          :resource_id    => record.resource_id,
+          :resource_type  => record.resource_type,
+          :account => CFO::Account.new(
+            :name     => record.account,
+            :service  => record.service,
+            :provider => record.provider,
+          ),
+          :billing_codes => codes.map {|c| [c.key, c.value]}
+        ).to_json
+      else
+        [404, {'error' => "Resource #{params[:resource_id]}"+
+          " in account #{params[:account_name]} not found"}.to_json
+        ]
+      end
+    end
+
+    ######## REPORTS ########
+
+    # Returns a cost report based on constraints in the request path.
+    get %r{^#{CFO.root}/report.*} do
+      constraints = constraints_from_path(request.path_info)
+      logger.info "Generating report for #{constraints.inspect}."
+      groupings = groupings_from_path(request.path_info)
+      logger.info "Using groupings #{groupings.inspect}."
+      report = CFO::Report.new(constraints, groupings, :logger => logger)
+      if params['content-type'] && (params['content-type'] =~ %r{(text/)?csv})
+        report_file = "report_"+
+          ((constraints.map {|c| c.flatten.flatten}).flatten).join('_')+
+          ".#{Time.now.strftime('%Y-%m-%d-%H%M%S')}.csv"
+        response['Content-Disposition'] = "attachment; filename=#{report_file}"
+        report.to_csv
+      else
+        report.to_json
+      end
+    end
+
+    CONSTRAINT_PATTERN  = /(for_\w+)/
+    GROUPING_PATTERN    = /by_(\w+)/
+
+    private
+
+    # Returns a Hash of report constraints (see Report.new) from a path String
+    def constraints_from_path(path)
+      constraints = []
+      tokens = (path.split('/')).drop(2)
+      while not tokens.empty?
+        if key_match = tokens.first.match(CONSTRAINT_PATTERN)
+          constraint_key, constraint_values = key_match[1], Array.new
+          constraint_values = tokens.drop(1).take_while do |t|
+            (t !~ CONSTRAINT_PATTERN) && (t !~ GROUPING_PATTERN)
+          end
+          constraints << {constraint_key => constraint_values}
+          tokens = tokens.drop(constraint_values.size + 1)
+        else
+          tokens = tokens.drop 1
+        end
+      end
+      constraints
+    end
+
+    # Returns a Hash of report constraints (see Report.new) from a path String
+    def groupings_from_path(path)
+      tokens = (path.split('/')).drop(2)
+      tokens.inject([]) do |groupings, token|
+        if not token.scan(GROUPING_PATTERN).empty?
+          groupings << token.scan(GROUPING_PATTERN).first.first
+        end
+        groupings
+      end
+    end
+
+    # start the server if ruby file executed directly
+    run! if app_file == $0
+  end
+end

data/lib/cfo/version.rb

@@ -0,0 +1,7 @@
+module CFO
+  # The version of this CFO Gem / Library.
+  VERSION = "0.1.0"
+
+  # This library's version of the CFO REST API.
+  API_VERSION = "1"
+end

data/lib/cloud_financial_officer.rb

@@ -0,0 +1,8 @@
+%w{rubygems bundler}.each {|lib| require lib}
+require File.join(File.dirname(__FILE__), "cfo/cfo")
+Bundler.require(:default, CFO.env.to_sym)
+Bundler.require(:development) if CFO.env == 'test'
+require 'cloud_cost_tracker'
+
+# Load all ruby files from 'cfo' directory
+Dir[File.join(File.dirname(__FILE__), "cfo/**/*.rb")].each {|f| require f}

data/lib/tasks/rspec.rake

@@ -0,0 +1,6 @@
+require 'rspec/core/rake_task'
+
+desc "Runs all tests"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = ['--color', '--format documentation', '-r ./spec/spec_helper.rb']
+end

data/lib/tasks/service.rake

@@ -0,0 +1,5 @@
+desc "Runs the service"
+task :service do
+  require './lib/cfo/service'
+  CFO::Service.run!
+end

data/spec/factories.rb

@@ -0,0 +1,35 @@
+ACCOUNT_NAME    = "fake-account-name"
+SERVICE_NAME    = "fake-service"
+PROVIDER_NAME   = "fake-provider"
+RESOURCE_ID     = "fake-resource-ID"
+RESOURCE_TYPE   = "fake-resource-type"
+BILLING_TYPE    = "fake-billing-type"
+RECORD_TOTAL    = 1.0
+RECORD_HOURLY   = 60.0
+RECORD_DURATION = 60
+
+FactoryGirl.define do
+  factory :account, :class => CFO::Account do
+    name     ACCOUNT_NAME
+    service  SERVICE_NAME
+    provider PROVIDER_NAME
+  end
+
+  factory :resource, :class => CFO::CloudResource do
+    resource_id    RESOURCE_ID
+    resource_type  RESOURCE_TYPE
+  end
+
+  factory :record, :class => CloudCostTracker::Billing::BillingRecord do
+    provider       PROVIDER_NAME
+    service        SERVICE_NAME
+    account        ACCOUNT_NAME
+    resource_id    RESOURCE_ID
+    resource_type  RESOURCE_TYPE
+    billing_type   BILLING_TYPE
+    start_time     Time.now - RECORD_DURATION
+    stop_time      Time.now
+    cost_per_hour  RECORD_HOURLY
+    total_cost     RECORD_TOTAL
+  end
+end

data/spec/lib/cfo/record_filter_spec.rb

@@ -0,0 +1,49 @@
+module CFO
+  include CloudCostTracker::Billing
+
+  describe RecordFilter do
+
+    # Number of BillingRecords to report on
+    NUM_RECORDS = 50  # Each has its own unique provider, service, etc.
+
+    before(:all) do
+      CloudCostTracker::Billing::BillingRecord.delete_all
+      @billing_start = Time.now
+      # Create many BillingRecords, each with its own unique attributes
+      (1..NUM_RECORDS).each do |i|
+        create(:record, {
+          :provider       => "#{PROVIDER_NAME}#{i}",
+          :service        => "#{SERVICE_NAME}#{i}",
+          :account        => "#{ACCOUNT_NAME}#{i}",
+          :resource_id    => "#{RESOURCE_ID}#{i}",
+          :resource_type  => "#{RESOURCE_TYPE}#{i}",
+          :billing_type   => "#{BILLING_TYPE}#{i}",
+        })
+      end
+      @billing_stop = Time.now + 1
+    end
+
+    after(:all) do
+      BillingRecord.delete_all
+    end
+
+    describe '#truncate_by_time' do
+      it "limits all records' start and stop times to within the given range" do
+        # make sure some records are out-of the start and stop range
+        first_record = BillingRecord.first
+        last_record = BillingRecord.last
+        first_record.start_time = @billing_start - SECONDS_PER_HOUR
+        last_record.stop_time = @billing_stop + SECONDS_PER_HOUR
+        [first_record, last_record].each {|r| r.save!}
+        # Now filter the records by time
+        results = RecordFilter.truncate_by_time(BillingRecord.all,
+           @billing_start, @billing_stop)
+        results.each do |truncated_record|
+          truncated_record.start_time.should >= @billing_start
+          truncated_record.stop_time.should <= @billing_stop
+        end
+      end
+    end
+
+  end
+end