cloud_cost_tracker 0.1.0

data/lib/cloud_cost_tracker/billing/compute/aws/servers.rb

@@ -0,0 +1,30 @@
+module CloudCostTracker
+  module Billing
+    module Compute
+      module AWS
+        # The default billing policy for Amazon EC2 server instances
+        class ServerBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_HOUR = YAML.load(File.read File.join(
+          CONSTANTS_DIR, 'compute-aws-servers', 'us-east-on_demand.yml'))
+
+          # Returns the storage cost for a given EC2 server
+          # over some duration (in seconds)
+          def get_cost_for_duration(ec2_server, duration)
+            return 0.0 if ec2_server.state =~ /(stopped|terminated)/
+            hourly_cost = CENTS_PER_HOUR[platform_for(ec2_server)][ec2_server.flavor_id]
+            (hourly_cost * duration) / SECONDS_PER_HOUR
+          end
+
+          # Returns either 'windows' or 'unix', based on this instance's platform
+          def platform_for(resource)
+            ('windows' == resource.platform) ? 'windows' : 'unix'
+          end
+
+          def billing_type ; "EC2 Instance runtime" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/compute/aws/snapshots.rb

@@ -0,0 +1,43 @@
+module CloudCostTracker
+  module Billing
+    module Compute
+      module AWS
+        # The default billing policy for Amazon EBS Snapshots
+        #
+        # *NOT REALLY WORKING* - AWS does not report the size
+        # of snapshots, so there's no way to accurately verify the cost!  :(
+        class SnapshotBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_GB_PER_MONTH = YAML.load(File.read File.join(
+          CONSTANTS_DIR, 'compute-aws-snapshots.yml'))
+
+          # Returns the storage cost for a given EBS Snapshot
+          # over some duration (in seconds)
+          # TODO - Make an estimate based on lineage and volume size
+          # This code is only accurate if the snapshot is the first for its volume
+          def get_cost_for_duration(snapshot, duration)
+            return 0.0  # TEMPORARILY DISABLED UNTIL WORKAROUND IS FOUND
+            CENTS_PER_GB_PER_MONTH[zone(snapshot)] * snapshot.volume_size *
+            duration / SECONDS_PER_MONTH
+          end
+
+          # Follows the snapshot's volume and returns its region, and
+          # chops the availability zone letter from the region
+          def zone(resource)
+            volume = resource.account_resources('volumes').find do |v|
+              v.identity == resource.volume_id
+            end
+            if volume && volume.availability_zone
+              volume.availability_zone.chop
+            else
+              "us-east-1"
+            end
+          end
+
+          def billing_type ; "EBS snapshot storage" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/compute/aws/volumes.rb

@@ -0,0 +1,30 @@
+module CloudCostTracker
+  module Billing
+    module Compute
+      module AWS
+        # The default billing policy for Amazon EBS Volumes
+        class VolumeBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_GB_PER_MONTH = YAML.load(File.read File.join(
+          CONSTANTS_DIR, 'compute-aws-volumes.yml'))
+
+          # Returns the storage cost for a given EBS Volume
+          # over some duration (in seconds)
+          def get_cost_for_duration(volume, duration)
+            return 0.0 if volume.state =~ /(deleting|deleted)/
+            CENTS_PER_GB_PER_MONTH[zone(volume)] * volume.size *
+            duration / SECONDS_PER_MONTH
+          end
+
+          # Chops the availability zone letter from the region
+          def zone(resource)
+            resource.availability_zone.chop
+          end
+
+          def billing_type ; "EBS volume storage" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/elasticache/clusters.rb

@@ -0,0 +1,33 @@
+module CloudCostTracker
+  module Billing
+    module AWS
+      module Elasticache
+        # The default billing policy for Amazon Elasticache Clusters
+        class ClusterBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_HOUR = YAML.load(File.read File.join(
+            CONSTANTS_DIR, 'elasticache-aws.yml'))
+
+          def billing_type ; "Elasticache Cluster runtime" end
+
+          # Returns the storage cost for a given Elasticache Cluster
+          # over some duration (in seconds)
+          def get_cost_for_duration(cluster, duration)
+            #@log.warn "Calculating cost for #{cluster.tracker_description}"
+            return 0.0 if cluster.status =~ /(stopped|terminated)/
+            hourly_cost = cluster.num_nodes *
+              CENTS_PER_HOUR[region(cluster)][cluster.node_type]
+            (hourly_cost * duration) / SECONDS_PER_HOUR
+          end
+
+          # Chops the availability zone letter from the availability zone
+          # and returns it as a string. e.g., "us-east-XXXXX" => "us-east"
+          def region(cluster)
+            cluster.zone.split('-')[0..1].join('-')
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/rds/server_storage.rb

@@ -0,0 +1,30 @@
+module CloudCostTracker
+  module Billing
+    module AWS
+      module RDS
+        # The default billing policy for Amazon RDS server storage costs
+        class ServerStorageBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_GB_PER_MONTH = YAML.load(File.read File.join(
+          CONSTANTS_DIR, 'rds-aws-servers', 'us-east-on_demand-storage.yml'))
+
+          # Returns the storage cost for a given RDS server
+          # over some duration (in seconds)
+          def get_cost_for_duration(rds_server, duration)
+            CENTS_PER_GB_PER_MONTH[zone_setting(rds_server)] *
+            rds_server.allocated_storage * duration / SECONDS_PER_MONTH
+          end
+
+          # returns either 'multi_az' or 'standard',
+          # depending on whether this RDS server is multi-AZ
+          def zone_setting(resource)
+            resource.multi_az ? 'multi_az' : 'standard'
+          end
+
+          def billing_type ; "RDS Instance storage" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/rds/servers.rb

@@ -0,0 +1,31 @@
+module CloudCostTracker
+  module Billing
+    module AWS
+      module RDS
+        # The default billing policy for Amazon RDS server runtime costs
+        class ServerBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_HOUR = YAML.load(File.read File.join(
+            CONSTANTS_DIR, 'rds-aws-servers', 'us-east-on_demand-mysql.yml'))
+
+          # Returns the runtime cost for a given RDS server
+          # over some duration (in seconds)
+          def get_cost_for_duration(rds_server, duration)
+            hourly_cost =
+              CENTS_PER_HOUR[zone_setting(rds_server)][rds_server.flavor_id]
+            (hourly_cost * duration) / SECONDS_PER_HOUR
+          end
+
+          # returns either 'multi_az' or 'standard',
+          # depending on whether this RDS server is multi-AZ
+          def zone_setting(resource)
+            resource.multi_az ? 'multi_az' : 'standard'
+          end
+
+          def billing_type ; "RDS Instance runtime" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/resource_billing_policy.rb

@@ -0,0 +1,119 @@
+# Abstract class, defines a generic Billing Policy that always returns 0.0 cost
+module CloudCostTracker
+  module Billing
+
+    PRECISION     = 10  # (Should match database migration precision)
+    # Defines a directory for holding YML pricing constants
+    CONSTANTS_DIR = File.join(File.dirname(__FILE__),'../../../config/billing')
+
+    # Some time and size constants
+
+    SECONDS_PER_MINUTE  = 60
+    SECONDS_PER_HOUR    = SECONDS_PER_MINUTE * 60
+    SECONDS_PER_DAY     = SECONDS_PER_HOUR * 24
+    SECONDS_PER_YEAR    = SECONDS_PER_DAY * 365
+    SECONDS_PER_MONTH   = SECONDS_PER_YEAR / 12
+    BYTES_PER_KB        = 1024
+    BYTES_PER_MB        = BYTES_PER_KB * 1024
+    BYTES_PER_GB        = BYTES_PER_MB * 1024
+
+    # Implements the logic for billing a single resource.
+    # All Billing Policies should inherit from this class, and define
+    # {#get_cost_for_duration}
+    class ResourceBillingPolicy
+      include CloudCostTracker
+
+      # Don't override this method - use {#setup} instead for
+      # one-time behavior
+      # @param [Hash] options optional parameters:
+      #  - :logger - a Ruby Logger-compatible object
+      def initialize(options={})
+        @log = options[:logger] || FogTracker.default_logger
+      end
+
+      # An initializer called by the framework once per billling cycle.
+      # Override this method if you need to perform high-latency operations,
+      # like network transactions, that should not be performed per-resource.
+      def setup(resources) ; end
+
+      # Returns the cost for a particular resource over some duration in seconds.
+      #  ALL BILLING POLICY SUBCLASSES SHOULD OVERRIDE THIS METHOD
+      # @param [Fog::Model] resource the resource to be billed
+      # @param [Integer] duration the number of seconds for this billing period.
+      # @return [Float] The amount the resource cost over duration seconds.
+      def get_cost_for_duration(resource, duration) ; 1.0 end
+
+      # Returns the default billing type for this policy.
+      # Override this to set a human-readable name for the policy.
+      # Defaults to the last part of the subclass name.
+      # @return [String] a description of the costs incurred under this policy.
+      def billing_type
+        self.class.name.split('::').last  #(defaluts to class name)
+      end
+
+      # Creates or Updates a BillingRecord for this BillingPolicy's resource.
+      # Don't override this -- it's called once for each resource by the
+      # {CloudCostTracker::Billing::AccountBillingPolicy}.
+      # @param [Fog::Model] resource the resource for the record to be written
+      # @param [Float] hourly_rate the resource's hourly rate for this period
+      # @param [Float] total the resource's total cost for this period
+      # @param [Time] start_time the start time for any new BillingRecords
+      # @param [Time] end_time the start time for any new BillingRecords
+      def write_billing_record_for(resource, hourly_rate, total,
+        start_time, end_time)
+        account         = resource.tracker_account
+        resource_type   = resource.class.name.split('::').last
+        return if total == 0.0  # Write no record if the cost is zero
+        new_record = BillingRecord.new(
+          :provider       => account[:provider],
+          :service        => account[:service],
+          :account        => account[:name],
+          :resource_id    => resource.identity,
+          :resource_type  => resource_type,
+          :billing_type   => billing_type,
+          :start_time     => start_time,
+          :stop_time      => end_time,
+          :cost_per_hour  => hourly_rate,
+          :total_cost     => total
+        )
+        new_record.set_codes(resource.billing_codes)
+        # Combine BillingRecords within maximim_gap of one another
+        write_or_merge_with_existing(new_record, account[:delay].to_i)
+      end
+
+      private
+
+      # Writes 'record' into the database if:
+      #   1. there's no previous record for this resource + billing type; or
+      #   2. the previous such record differs in hourly cost or billing codes; or
+      #   3. the records are separated by more than maximum_time_gap
+      # Otherwise, `record` is merged with the previous record that matches its
+      # resource and billing type
+      def write_or_merge_with_existing(new_record, maximum_time_gap)
+        write_new_record = true
+        last_record = BillingRecord.most_recent_like(new_record)
+        # If the previous record for this resource/billing type has the same
+        # hourly rate and billing codes, just update the previous record
+        if last_record && last_record.overlaps_with(new_record, maximum_time_gap)
+          if (last_record.cost_per_hour.round(PRECISION) ==
+              new_record.cost_per_hour.round(PRECISION)) &&
+              (last_record.billing_codes == new_record.billing_codes)
+            @log.debug "Updating record #{last_record.id} for "+
+              " #{new_record.resource_type} #{new_record.resource_id}"+
+              " in account #{new_record.account}"
+            last_record.merge_with new_record
+            write_new_record = false
+          else  # If the previous record has different rate or codes...
+            # Make the new record begin where the previous one leaves off
+            new_record.start_time = last_record.stop_time
+          end
+        end
+        if write_new_record
+          @log.debug "Creating new record for for #{new_record.resource_type}"+
+              " #{new_record.resource_id} in account #{new_record.account}"
+          ActiveRecord::Base.connection_pool.with_connection {new_record.save!}
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/billing/storage/aws/directories.rb

@@ -0,0 +1,47 @@
+module CloudCostTracker
+  module Billing
+    module Storage
+      module AWS
+        # The default billing policy for Amazon S3 buckets
+        class DirectoryBillingPolicy < ResourceBillingPolicy
+          # The YAML pricing data is read from config/billing
+          CENTS_PER_GB_PER_MONTH = YAML.load(File.read File.join(
+          CONSTANTS_DIR, 'storage-aws-directories.yml'))
+
+          # Returns the runtime cost for a given S3 bucket
+          # over some duration (in seconds)
+          def get_cost_for_duration(s3_bucket, duration)
+            CENTS_PER_GB_PER_MONTH[zone(s3_bucket)] * total_size(s3_bucket) *
+            duration / SECONDS_PER_MONTH
+          end
+
+          # Chops the availability zone letter from the region
+          def zone(resource)
+            'us-east-1'
+          end
+
+          # Returns the total size of all a bucket's objecs, in GB
+          def total_size(bucket)
+            # check for saved value
+            return @bucket_size[bucket] if @bucket_size[bucket]
+            @log.debug "Computing size for #{bucket.tracker_description}"
+            total_bytes = 0
+            bucket.files.each {|object| total_bytes += object.content_length}
+            @log.debug "total bytes = #{total_bytes}"
+            # save the total size for later
+            @bucket_size[bucket] = total_bytes / BYTES_PER_GB.to_f
+          end
+
+          # Remembers each bucket size, because iterating over S3 objects is
+          # slow, and get_cost_for_duration is called twice
+          def setup(resources)
+            @bucket_size = Hash.new
+          end
+
+          def billing_type ; "S3 Bucket storage" end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/account_coding_policy.rb

@@ -0,0 +1,73 @@
+module CloudCostTracker
+  module Coding
+    # Implements the generic logic for attaching billing codes to all resources
+    # in a single account.
+    # Initializes the necessary ResourceCodingPolicy objects, sorts the
+    # resources by policy, and calls {#code} once on each resource's policy,
+    # to compute the billing codes as pairs of Strings.
+    class AccountCodingPolicy
+
+      # Creates an object that implements a coding policy
+      # that attaches no billing codes
+      # @param [Array <Fog::Model>] resources the resources to code
+      # @param [Hash] options optional parameters:
+      #  - :logger - a Ruby Logger-compatible object
+      def initialize(resources, options={})
+        @log = options[:logger] || FogTracker.default_logger
+        setup_resource_coding_agents(resources)
+      end
+
+      # An initializer called by the framework once per bill coding cycle.
+      # Override this method if you need to perform high-latency operations,
+      # like network transactions, that should not be performed per-resource.
+      def setup(resources) ; end
+
+      # Defines the order in which resource collections are coded.
+      # Override this method if you need to code the resource collections
+      # in a particular order. Return an Array of the Fog::Model subclasses.
+      # @return [Array <Class>] the class names, in preferred coding order
+      def priority_classes ; Array.new end
+
+      # Defines an acount-wide coding strategy for coding each resource.
+      # Override this method if you need to write logic for attaching billing
+      # codes to all resources in an account, regardless of collection / type.
+      def attach_account_codes(resource) ; end
+
+      # Defines the default method for coding all resources.
+      # Attaches Billing Codes (String pairs) to resources, as @billing_codes.
+      # Resources whose class is in {#priority_classes} are coded first.
+      def code(resources)
+        return if resources.empty?
+        account = resources.first.tracker_account
+        @log.info "Coding account #{account[:name]}"
+        resources.each {|resource| attach_account_codes(resource)}
+        classes_to_code = priority_classes + (@agents.keys - priority_classes)
+        classes_to_code.delete_if {|res_class| @agents[res_class] == nil}
+        classes_to_code.each do |fog_model_class|
+          @log.info "Coding #{fog_model_class} in account #{account[:name]}"
+          collection = resources.select {|r| r.class == fog_model_class}
+          collection.each do |resource|
+            @agents[fog_model_class].each do |agent|
+              @log.debug "Coding #{resource.tracker_description}"
+              agent.code(resource)
+            end
+          end
+        end
+      end
+
+      private
+
+      # Builds a Hash of CodingPolicy agents, indexed by resource Class name
+      def setup_resource_coding_agents(resources)
+        @agents = Hash.new
+        ((resources.collect {|r| r.class}).uniq).each do |resource_class|
+          @agents[resource_class] = CloudCostTracker::create_coding_agents(
+            resource_class, {:logger => @log})
+          @agents[resource_class].each {|agent| agent.setup(resources)}
+          @agents.delete(resource_class) if @agents[resource_class].empty?
+        end
+      end
+
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/compute/aws/account_coding_policy.rb

@@ -0,0 +1,38 @@
+require 'cloud_cost_tracker/coding/account_coding_policy'
+module CloudCostTracker
+  module Coding
+    module Compute
+      module AWS
+        # Implements the logic for attaching billing codes to all resources
+        # in a single AWS EC2 account, and defines the default order in which
+        # EC2 resources get coded.
+        class AccountCodingPolicy < CloudCostTracker::Coding::AccountCodingPolicy
+
+          # Defines the order in which EC2 resources are coded.
+          # @return [Array <Class>] the class names, in preferred coding order
+          def priority_classes
+            [
+              Fog::Compute::AWS::Tag,
+              Fog::Compute::AWS::SecurityGroup,
+              Fog::Compute::AWS::KeyPair,
+              Fog::Compute::AWS::Server,    # pulls codes from security group
+              Fog::Compute::AWS::Volume,    # pulls codes from server
+              Fog::Compute::AWS::Snapshot,  # pulls codes from volume
+              Fog::Compute::AWS::Address,
+            ]
+          end
+
+          # Translates all AWS Tags into Billing Codes
+          def attach_account_codes(resource)
+            if resource.respond_to?(:tags) and resource.tags
+              resource.tags.each do |key, value|
+                resource.code(key, value)
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/compute/aws/servers.rb

@@ -0,0 +1,26 @@
+module CloudCostTracker
+  module Coding
+    module Compute
+      module AWS
+        # Implements the logic for attaching billing codes to EC2 servers
+        class ServerCodingPolicy < ResourceCodingPolicy
+
+          # Copies all billing codes from this server's Security Group
+          def code(ec2_server)
+            ec2_server.groups.each do |group_name|
+              group = ec2_server.account_resources('security_groups').find do |g|
+                g.identity == group_name
+              end
+              if group
+                group.billing_codes.each do |billing_code|
+                  ec2_server.code(billing_code[0], billing_code[1])
+                end
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/compute/aws/volumes.rb

@@ -0,0 +1,26 @@
+module CloudCostTracker
+  module Coding
+    module Compute
+      module AWS
+        # Implements the logic for attaching billing codes to EBS Volumes
+        class VolumeCodingPolicy < ResourceCodingPolicy
+
+          # Copies all billing codes from any attached instance to this volume
+          def code(ebs_volume)
+            if (ebs_volume.state == "in-use") && (ebs_volume.server_id != "")
+              server = ebs_volume.account_resources('servers').find do |server|
+                server.identity == ebs_volume.server_id
+              end
+              if server
+                server.billing_codes.each do |server_code|
+                  ebs_volume.code(server_code[0], server_code[1])
+                end
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/rds/account_coding_policy.rb

@@ -0,0 +1,25 @@
+module CloudCostTracker
+  module Coding
+    module RDS
+      module AWS
+        # Implements the logic for attaching billing codes to all resources
+        # in a single AWS RDS account, and defines the default order in which
+        # RDS resources get coded.
+        class AccountCodingPolicy < CloudCostTracker::Coding::AccountCodingPolicy
+
+          # Defines the order in which EC2 resources are coded.
+          # @return [Array <Class>] the class names, in preferred coding order
+          def priority_classes
+            [
+              Fog::AWS::RDS::SecurityGroup,
+              Fog::AWS::RDS::ParameterGroup,
+              Fog::AWS::RDS::Server,    # pulls codes from security group
+              Fog::AWS::RDS::Snapshot,  # pulls codes from server
+            ]
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/rds/servers.rb

@@ -0,0 +1,59 @@
+module CloudCostTracker
+  module Coding
+    module AWS
+      module RDS
+        # Implements the logic for attaching billing codes to RDS instances
+        class ServerCodingPolicy < ResourceCodingPolicy
+
+          # Indexes the security and parameter groups, so that BillingCodes
+          # can be pulled from them quickly when code() is called
+          def setup(resources)
+            @acc_sec_groups = Hash.new    # Index the security groups
+            resources.first.account_resources('security_groups').each do |group|
+              @acc_sec_groups[group.identity] = group
+            end
+            @acc_param_groups = Hash.new  # Index the parameter groups
+            resources.first.account_resources('parameter_groups').each do |group|
+              @acc_param_groups[group.identity] = group
+            end
+          end
+
+          # Copies all billing codes from this server's Groups
+          def code(rds_server)
+            code_from_security_group(rds_server)
+            code_from_parameter_group(rds_server)
+          end
+
+          # Copies all billing codes from this server's Security Group
+          def code_from_security_group(rds_server)
+            if rds_server.db_security_groups
+              server_groups = rds_server.db_security_groups.map do |group|
+                @acc_sec_groups[group['DBSecurityGroupName']]
+              end
+              server_groups.each do |group|
+                group.billing_codes.each do |billing_code|
+                  rds_server.code(billing_code[0], billing_code[1])
+                end
+              end
+            end
+          end
+
+          # Copies all billing codes from this server's Parameter Group
+          def code_from_parameter_group(rds_server)
+            if rds_server.db_parameter_groups
+              server_groups = rds_server.db_parameter_groups.map do |group|
+                @acc_param_groups[group['DBParameterGroupName']]
+              end
+              server_groups.each do |group|
+                group.billing_codes.each do |billing_code|
+                  rds_server.code(billing_code[0], billing_code[1])
+                end
+              end
+            end
+          end
+
+        end
+      end
+    end
+  end
+end

data/lib/cloud_cost_tracker/coding/resource_coding_policy.rb

@@ -0,0 +1,25 @@
+# Abstract class, defines a generic resource Coding Policy that attaches no codes
+module CloudCostTracker
+  module Coding
+    class ResourceCodingPolicy
+
+      # Creates an object that implements a billing policy
+      # that attaches no billing codes
+      # @param [Hash] options optional parameters:
+      #  - :logger - a Ruby Logger-compatible object
+      def initialize(options={})
+        @log = options[:logger] || FogTracker.default_logger
+      end
+
+      # Used by subclasses to perform setup each time an account's
+      # resources are coded
+      # High-latency operations like network transactions that are not
+      # per-resource should be performed here
+      def setup(resources) ; end
+
+      # Attaches Billing Codes (String pairs) to resource, as billing_codes
+      def code(resource) ; end
+
+    end
+  end
+end