pal_tool 0.2.1

data/lib/pal/operation/projection.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+require "pal"
+
+module Pal
+  module Operation
+    class Projection
+      include Log
+
+      # @return [String]
+      attr_reader :type
+
+      # @return [String]
+      attr_reader :property
+
+      # @param [String] type
+      # @param [String] property
+      def initialize(type, property)
+        @type = type
+        @property = property
+      end
+
+      # @param [Array<String>] group_by_rules # export columns
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def process(group_by_rules, groups, column_headers)
+        log_info("Calling down to projection impl [#{type}]")
+        _process_impl(group_by_rules, groups, column_headers)
+      end
+
+      # @return [Boolean]
+      def processable?
+        !(@type.nil? || @property.nil?)
+      end
+
+      private
+
+      # @abstract
+      # @param [Array<String>] _group_by_rules
+      # @param [Hash] _groups
+      # @param [Hash] _column_headers
+      # @return [Array] rows, column_headers
+      def _process_impl(_group_by_rules, _groups, _column_headers)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+
+    class SumProjectionImpl < Projection
+      # @param [String] property
+      def initialize(property)
+        super("sum", property)
+      end
+
+      private
+
+      # @param [Array<String>] group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      def _process_impl(group_by_rules, groups, column_headers)
+        rows = []
+        sum_column_idx = column_headers[@property]
+
+        raise "Missing column. Please include [#{@property}] in columns #{column_headers.keys}." unless sum_column_idx
+
+        groups.each_key do |key|
+          sum = 0.0
+
+          row = groups[key]
+
+          row.each do |entry|
+            sum += entry[sum_column_idx].to_f
+          end
+
+          arr = []
+          group_by_rules.each do |gb|
+            idx = column_headers[gb]
+            arr << row[0][idx]
+          end
+
+          arr << sum.round(8)
+          rows << arr
+        end
+
+        column_headers = {}
+        group_by_rules.each_with_index { |gb, idx| column_headers[gb] = idx }
+        column_headers["sum_#{@property}"] = group_by_rules.size
+
+        [rows, column_headers]
+      end
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/AbcSize
+    end
+
+    class DistinctProjectionImpl < Projection
+      # @param [String] property
+      def initialize(property)
+        super("distinct", property)
+      end
+
+      private
+
+      # @param [Array<String>] _group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def _process_impl(_group_by_rules, groups, column_headers)
+        rows = []
+        distinct_column_idx = column_headers[@property]
+
+        raise "Missing column index. Please include [#{@property}] in column extraction." unless distinct_column_idx
+
+        groups.each_key do |key|
+          row = groups[key]
+
+          row.each do |entry|
+            prop = entry[distinct_column_idx]
+            rows << prop unless rows.include?(prop)
+          end
+        end
+
+        column_headers = {}
+        column_headers["distinct_#{@property}"] = 0
+
+        [rows.map { |x| [x] }, column_headers]
+      end
+    end
+
+    class MaxMinProjectionImpl < Projection
+      private
+
+      # @param [Array<String>] _group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      # rubocop:disable Metrics/AbcSize
+      def _process_impl(_group_by_rules, groups, column_headers)
+        max_vals = {}
+        max_column_idx = column_headers[@property]
+
+        groups.each_key do |key|
+          row = groups[key]
+
+          row.each do |entry|
+            prop_val = entry[max_column_idx].to_f
+
+            max_vals[key] = entry unless max_vals.key?(key)
+            max_vals[key] = entry if _comparator_proc.call(max_vals[key][0][max_column_idx].to_f, prop_val)
+          end
+        end
+
+        rows = max_vals.values
+
+        new_column_headers = {}
+        column_headers.keys.each_with_index { |ch, idx| new_column_headers[ch] = idx }
+
+        [rows, new_column_headers]
+      end
+      # rubocop:enable Metrics/AbcSize
+      #
+
+      def _comparator_proc
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+
+    class MaxProjectionImpl < MaxMinProjectionImpl
+      # @param [String] property
+      def initialize(property)
+        super("max", property)
+      end
+
+      private
+
+      def _comparator_proc
+        proc { |x, y| x < y }
+      end
+    end
+
+    class MinProjectionImpl < MaxMinProjectionImpl
+      # @param [String] property
+      def initialize(property)
+        super("min", property)
+      end
+
+      private
+
+      def _comparator_proc
+        proc { |x, y| x > y }
+      end
+    end
+
+    class DefaultProjectionImpl < Projection
+      # @param [String] property
+      def initialize(property)
+        super("default", property)
+      end
+
+      # @param [Array<String>] _group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def _process_impl(_group_by_rules, groups, column_headers)
+        [groups.values, column_headers]
+      end
+    end
+
+    class AverageProjectionImpl < Projection
+      # @param [String] property
+      def initialize(property)
+        super("average", property)
+      end
+
+      private
+
+      # @param [Array<String>] group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      def _process_impl(group_by_rules, groups, column_headers)
+        rows = []
+        sum_column_idx = column_headers[@property]
+
+        groups.each_key do |key|
+          sum = 0.0
+
+          records = groups[key]
+          records.each do |entry|
+            sum += entry[sum_column_idx].to_f
+          end
+
+          arr = []
+          group_by_rules.each do |gb|
+            next if gb.eql? @property
+
+            idx = column_headers[gb]
+            arr << records[0][idx]
+          end
+
+          arr << sum.round(8) / records.size
+          rows << arr
+        end
+
+        column_headers = {}
+        group_by_rules.each_with_index do |gb, idx|
+          next if gb.eql? @property
+
+          column_headers[gb] = idx
+        end
+
+        column_headers["average_#{@property}"] = group_by_rules.size - 1 # remove default
+
+        [rows, column_headers]
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+    end
+
+    class CountProjectionImpl < Projection
+      # @param [String] property
+      def initialize(property)
+        super("count", property)
+      end
+
+      private
+
+      # @param [Array<String>] _group_by_rules
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      # rubocop:disable Metrics/AbcSize
+      def _process_impl(_group_by_rules, groups, column_headers)
+        distinct_column_idx = column_headers[@property]
+        raise "Missing column index. Please include [#{@property}] in column extraction." unless distinct_column_idx
+
+        count_map = {}
+
+        groups.each_key do |key|
+          groups[key].each do |entry|
+            prop = entry[distinct_column_idx]
+
+            count_map[prop] = 0 unless count_map[prop]
+            count_map[prop] += 1
+          end
+        end
+
+        column_headers = {}
+        column_headers[@property] = 0
+        column_headers["count_#{@property}"] = 0
+
+        [count_map.map { |k, v| [k, v] }, column_headers]
+      end
+      # rubocop:enable Metrics/AbcSize
+    end
+  end
+end
+

data/lib/pal/plugin.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "pal"
+require "pal/log"
+
+module Pal
+  module Plugin
+    include Log
+
+    def register_plugins(plugin_dir="plugins")
+      log_info "Registering any plugins for directory [#{plugin_dir}]"
+      PluginManager.instance.register(plugin_dir)
+    end
+
+    class PluginManager
+      include Singleton
+      include Log
+
+      def register(plugin_dir)
+        candidates = Dir.glob("#{plugin_dir}/**/*.rb").select { |e| File.file? e }
+
+        log_info "Found a total of [#{candidates.size}] candidates"
+        candidates.each do |file_path|
+          full_clazz_name = get_clazz_name(file_path)
+
+          next unless defined?(full_clazz_name)
+          next unless full_clazz_name.start_with?("Pal::")
+
+          log_info "[#{full_clazz_name}] has passed validation and will be loaded"
+
+          load file_path
+
+          validate_plugin(full_clazz_name)
+        end
+      end
+
+      private
+
+      def get_clazz_name(file_path)
+        mod, clazz = file_path.split("/")[-2..]
+        full_clazz = clazz.split("_").map(&:capitalize).join("").gsub(".rb", "")
+        "Pal::#{mod.capitalize}::#{full_clazz}"
+      end
+
+      # @param [String] full_clazz_name
+      # @raise [RuntimeError]
+      def validate_plugin(full_clazz_name)
+        clazz_ins = Kernel.const_get(full_clazz_name)
+        ancestors = clazz_ins.ancestors
+
+        valid_candidates = [Pal::Operation::BaseExportHandler]
+        unless valid_candidates.find { |a| ancestors.include?(a) }
+          log_error("Invalid plugin has been given. Valid plugin candidates are: #{valid_candidates.inspect}")
+          raise "Invalid plugin given!"
+        end
+
+        true
+      end
+    end
+  end
+end

data/lib/pal/request/metadata.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Pal
+  module Request
+    class Metadata
+      include Pal::ObjectHelpers
+
+      # @return [String]
+      attr_accessor :version, :name, :description, :handler
+
+      def initialize(opts={})
+        @description = opts["description"]
+        @version = opts["version"]
+        @handler = opts["handler"]
+        @name = opts["name"]
+      end
+    end
+  end
+end

data/lib/pal/request/runbook.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "pal/operation/filter_evaluator"
+require "pal/operation/exporter"
+require "pal/operation/actions"
+require "pal/request/metadata"
+require "pal/common/object_helpers"
+require "json"
+
+module Pal
+  module Request
+    class Runbook
+      include ObjectHelpers
+
+      # @return [Pal::Request::Metadata]
+      attr_reader :metadata
+
+      # @return [Pal::Operation::FilterEvaluator]
+      attr_reader :filters
+
+      # @return [Pal::Operation::Exporter]
+      attr_reader :exporter
+
+      # @return [Pal::Operation::Actions]
+      attr_reader :actions
+
+      # @return [Pal::Operation::Transforms]
+      attr_reader :transforms
+
+      # @return [Hash]
+      attr_accessor :column_overrides
+
+      # @param [Array<Hash>] opts
+      def filters=(opts)
+        @filters = Pal::Operation::FilterEvaluator.new(opts)
+      end
+
+      # @param [Hash] opts
+      def metadata=(opts)
+        @metadata = Pal::Request::Metadata.new(opts)
+      end
+
+      # @param [Hash] opts
+      def exporter=(opts)
+        @exporter = Pal::Operation::Exporter.new.from_hash(opts)
+      end
+
+      # # @param [Hash] opts
+      # def transforms=(opts)
+      #   @transforms = Pal::Operation::Transforms.new(opts)
+      # end
+    end
+  end
+end

data/lib/pal/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Pal
+  VERSION = "0.2.1"
+end

data/lib/pal.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "pal/version"
+require "pal/main"
+require "pal/configuration"
+require "pal/log"
+
+require "pal/operation/filter_evaluator"
+require "pal/handler/processor"
+require "pal/operation/exporter"
+require "pal/operation/actions"
+
+require "pal/request/runbook"
+require "pal/handler/manager"
+
+require "logger"
+
+# Entry point for Pal services
+module Pal
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= Logger.new($stdout).tap do |log|
+        log.progname = name
+      end
+    end
+  end
+
+  # Exception classes
+  class ValidationError < StandardError
+    attr_reader :errors
+
+    def initialize(errors, msg="Invalid Request")
+      super(msg)
+      @errors = errors
+    end
+
+    def message
+      "Validation error: [#{@errors.join(", ")}]"
+    end
+  end
+end

data/plugins/PLUGINS.md

@@ -0,0 +1 @@
+## Plugins Documentation

data/plugins/operation/terminal_exporter_impl.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "pal"
+require "pal/operation/exporter"
+
+module Pal
+  module Operation
+    class TerminalExporterImpl < BaseExportHandler
+      def _export(rows, _column_headers)
+        puts "Inside plugin! You passed me [#{rows.size}] rows"
+      end
+    end
+  end
+end

data/templates/DOCUMENTATION.md

@@ -0,0 +1,46 @@
+Documentation for Templates
+===========================
+
+Templates are made up of three parts - metadata, filters and exporter. At is most simplest, only the metadata field needs to be provided.
+
+##Metadata
+```json
+  "metadata" : {
+    "version" : "2022-04-02",
+    "name" : "<name>",
+    "description" : "<description>"
+    "handler" : "<handler>"
+  },...
+```
+
+###Keys
+- **version**: API version, default is [2022-04-02]
+- **name**: Name of the template.
+- **description**: Description of the template.
+- **handler**: Specifies the type of spreadsheet to process. Default is [AwsCur]
+
+-------------------------------------------
+
+##Filters
+```json
+  ...
+  "filters":  {
+    "condition": "<and/or>",
+    "rules": [{
+        "field": "<column_header>",
+        "type": "<number|string>",
+        "operator": "<operator>",
+        "value": "<value>"
+      }
+    ]
+  },...
+```
+###Keys
+- **condition**: Must be either and/or. Can be nested.
+- **rules**: List of rules
+  - **field**: Field found in spreadsheet.
+  - **type**: Data type, either string or number.
+  - **operator**: Predicate to validate against - <equal, not_equal,..>.
+  - **value**: Value to validate against.
+  
+  

data/templates/aws/data_transfer/data_transfer_breakdown.json

@@ -0,0 +1,93 @@
+{
+  "metadata" : {
+    "version" : "2022-04-02",
+    "name" : "Data Transfer Breakdown",
+    "handler" : "AwsCur",
+    "description" : "Spend breakdown for data transfer (From/To)"
+  },
+  "filters": {
+    "condition": "AND",
+    "rules": [
+      {
+        "field": "product/productFamily",
+        "type": "string",
+        "operator": "equal",
+        "value": "Data Transfer"
+      },
+      {
+        "condition": "OR",
+        "rules": [
+          {
+            "field": "lineItem/LineItemType",
+            "type": "string",
+            "operator": "equal",
+            "value": "Usage"
+          },
+          {
+            "field": "lineItem/LineItemType",
+            "type": "string",
+            "operator": "equal",
+            "value": "DiscountedUsage"
+          },
+          {
+            "field": "lineItem/LineItemType",
+            "type": "string",
+            "operator": "equal",
+            "value": "SavingsPlanCoveredUsage"
+          }
+        ]
+      }
+    ]
+  },
+  "exporter" : {
+    "types" : [{
+      "name" : "table",
+      "settings" : {
+        "title" : "Data Transfer by Product and Type"
+      }
+    }],
+    "properties" : [
+      "lineItem/UsageStartDate",
+      "lineItem/UsageAccountId",
+      "lineItem/ProductCode",
+      "lineItem/ResourceId",
+      "lineItem/UsageType",
+      "product/fromLocation",
+      "product/toLocation",
+      "product/productFamily",
+      "lineItem/UnblendedCost"
+    ],
+    "actions" : {
+      "group_by" : [
+        "lineItem/ProductCode",
+        "lineItem/UsageAccountId",
+        "lineItem/UsageType",
+        "product/fromLocation",
+        "product/toLocation",
+        "product/productFamily"
+      ],
+      "sort_by" : "lineItem/ProductCode",
+      "projection" : {
+        "type" : "sum",
+        "property" : "lineItem/UnblendedCost"
+      }
+    }
+  },
+  "column_overrides" : {
+    "lineItem/UsageStartDate" : {
+      "data_type": "date"
+    }
+  },
+  "__comments__" : {
+    "optional": {
+      "actions" : {
+        "group_by" : [
+          "lineItem/UsageStartDate",
+          "lineItem/ResourceId"
+        ]
+      }
+    }
+  }
+}
+
+