pal_tool 0.2.1

data/lib/pal/common/safe_hash_parse.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "pal"
+require "json"
+require "jsonpath"
+
+module Pal
+  # The most lazy way to find things in hashes and JSON.
+  # Provides default and optional params
+  # Provides safe navigation with hash key dot notation ('this.is.a.key')
+  class SafeHashParse
+    class << self
+      # @param [String] json_str
+      # @param [Object] key
+      # @param [Boolean] optional
+      # @param [Object] default
+      # @return [Array]
+      def extract_from_json(json_str, key, optional=false, default=nil)
+        val = JsonPath.new(key.to_s).on(json_str)
+        return val if val && !val.empty?
+        return [] unless optional
+
+        [default]
+      rescue JSON::ParserError, MultiJson::ParseError, ArgumentError => e
+        raise e unless optional
+
+        [default]
+      end
+
+      # @param [Hash] hash
+      # @param [Object] search_key
+      # @param [Boolean] optional
+      # @param [Object, nil] default
+      # @return [Object, nil]
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      def extract_from_hash(hash, search_key, optional=false, default=nil)
+        keys = format_key(search_key)
+        last_level = hash
+        searched = nil
+
+        keys.each_with_index do |key, index|
+          break unless last_level.is_a?(Hash) && last_level.key?(key.to_s)
+
+          if index + 1 == keys.length
+            searched = last_level[key.to_s] || last_level[key.to_sym]
+          else
+            last_level = last_level[key.to_s] || last_level[key.to_sym]
+          end
+        end
+
+        return searched if searched
+        return nil unless optional
+
+        default
+      end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      # @param [Object] key
+      # @return [Array]
+      def format_key(key)
+        return [key.downcase] if key.is_a?(Symbol)
+
+        if key.is_a?(String)
+          return [key.downcase.to_sym] unless key.include?(".")
+
+          return key.to_s.split(".").map { |s| s.downcase.to_sym }
+        end
+
+        raise ArgumentError, "Key [#{key}] must be either a String or Symbol"
+      end
+
+      # @param [String] json
+      def all_values_from_json(json)
+        all_values(JSON.parse(json))
+      end
+
+      # @param [Hash] hash
+      def all_values(hash)
+        hash.flat_map { |_k, v| (v.is_a?(Hash) ? all_values(v) : [v]) }
+      end
+    end
+  end
+end

data/lib/pal/configuration.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "singleton"
+require "json"
+require "pal/log"
+
+module Pal
+  # Configuration management module for Pal
+  module Configuration
+    include Log
+
+    # @return [Config]
+    def config
+      conf = ConfigurationSource.instance.config
+      raise "Set config first" unless conf
+
+      conf
+    end
+
+    # @param [Config] request_config
+    def register_config(request_config)
+      log_info "Setting config"
+      ConfigurationSource.instance.load_config(request_config)
+    end
+
+    # Config data class - holds configuration settings.
+    class Config
+      attr_accessor :source_file_loc, :template_file_loc, :output_dir
+
+      # @return [Boolean]
+      def validate
+        errors = decorate_errors
+
+        if errors.size.positive?
+          errors.each { |x| Pal.logger.info x }
+          raise Pal::ValidationError.new(errors, "Invalid request.")
+        end
+
+        true
+      end
+
+      # Can probably remove this
+      def read_template_file
+        JSON.parse(File.read(@template_file_loc))
+      end
+
+      def all_source_files
+        @source_file_loc.split(",")
+      end
+
+      private
+
+      def decorate_errors
+        # Add directory validation
+        # Check billing file is a valid billing file
+        errors = []
+        errors << "Missing property: template file [-t]." unless @template_file_loc
+        errors << "Missing property: input file [-s]." unless @source_file_loc
+        errors << "File not found: billing file must exist" unless File.exist?(@source_file_loc || "")
+        errors
+      end
+    end
+
+    # Config storage source for access, stored as singleton.
+    class ConfigurationSource
+      include Singleton
+
+      attr_reader :config
+
+      # @param [Config] config
+      # @return [Config]
+      def load_config(config)
+        @config = config
+      end
+    end
+  end
+end

data/lib/pal/handler/base.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "pal"
+
+module Pal
+  module Handler
+    class Base
+      include Configuration
+      include Log
+
+      # @param [Pal::Request::Runbook] runbook
+      def initialize(runbook)
+        @runbook = runbook
+      end
+
+      # @return [Operation::ProcessorContext]
+      # rubocop:disable Metrics/AbcSize
+      def process_runbook
+        log_debug("Processing runbook started, setting up context.")
+        ctx = Operation::ProcessorContext.new
+        ctx.column_type_definitions = retrieve_column_definitions
+
+        # Get CSV parser
+        # Each impl needs to return a hash of candidate columns and values
+        # Extract headers
+        # Extract values
+
+        log_debug("Calling off to parse impl for CSV processing.")
+
+        # Different impls may choose to stream file, so we hand in a location and let it decide.
+
+        config.all_source_files.each_with_index do |file, idx|
+          log_info "Opening file [#{file}][#{idx}]"
+
+          _parse_file(ctx, _csv_processor(file)) do |row|
+            ctx.add_candidate(row) if should_include?(@runbook.filters, row, ctx.column_headers)
+          end
+        end
+
+        log_info "Process completed with #{ctx.candidates.size} candidate records found."
+
+        ctx
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      # @return [Boolean]
+      # @param [Pal::Operation::FilterEvaluator] filter_eval
+      # @param [Array] row
+      # @param [Hash] column_headers
+      def should_include?(filter_eval, row, column_headers)
+        return true unless filter_eval
+
+        filter_eval.test_property(row, column_headers)
+      end
+
+      # @return [Hash, nil]
+      def retrieve_column_definitions
+        overrides = @runbook.column_overrides || {}
+        path = File.join(File.dirname(__FILE__), "definitions/#{_type}.json")
+
+        return overrides unless File.exist?(path)
+
+        default_defs = JSON.parse(File.read(path))
+        default_defs.merge(overrides)
+      end
+
+      protected
+
+      # @abstract
+      # @param [ProcessorContext] _ctx
+      # @param [CSVProcessor] _processor
+      # @param [Proc] _block
+      # @return [Hash]
+      def _parse_file(_ctx, _processor, &_block)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # @abstract
+      # @param [String] _source_file_loc
+      # @return [CSVProcessor]
+      def _csv_processor(_source_file_loc)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # @abstract
+      # @return [String]
+      def _type
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def _extract_headers; end
+    end
+
+    # Generic has first row column headers, then data rows.
+    class GenericCSVHandlerImpl < Base
+      include Log
+
+      # @param [ProcessorContext] ctx
+      # @param [Pal::Operation::CSVProcessor] csv_processor
+      # @param [Proc] _block
+      # @return [Hash]
+      # ---
+      # Each impl needs to return a hash of candidate columns and values
+      # eg. { col_name: col_value, col_name_2: col_value_2 }
+      def _parse_file(ctx, csv_processor, &_block)
+        log_info("Starting to process file, using #{csv_processor.class} processor for #{_type} CUR file.")
+        ctx.current_file_row_count = 0
+
+        csv_processor.parse(ctx, header: :none) do |row|
+          if ctx.current_file_row_count == 1
+            ctx.extract_column_headers(row)
+            next
+          end
+
+          yield row
+        end
+      end
+
+      # @param [String] source_file_loc
+      # @return [Pal::Operation::CSVProcessor]
+      def _csv_processor(source_file_loc)
+        Operation::CSVProcessor.retrieve_default_processor(source_file_loc)
+      end
+
+      # @return [String]
+      def _type
+        "generic"
+      end
+    end
+
+    class AwsCurHandlerImpl < GenericCSVHandlerImpl
+      # @return [String]
+      def _type
+        "aws_cur"
+      end
+    end
+  end
+end

data/lib/pal/handler/definitions/aws_cur.json

@@ -0,0 +1,8 @@
+{
+  "lineItem/UsageStartDate" : {
+    "data_type": "datetime"
+  },
+  "lineItem/BlendedCost" : {
+    "data_type": "decimal"
+  }
+}

data/lib/pal/handler/manager.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Pal
+  module Handler
+    class Manager
+      include Log
+
+      attr_accessor :handler
+
+      # @param [Base] handler
+      def initialize(handler)
+        raise TypeError.new("Service must be type of BaseServiceImpl") unless handler.is_a? Base
+
+        @handler = handler
+      end
+
+      # @param [Pal::Request::Runbook] runbook
+      # @return [Array, Hash]
+      def process_runbook(runbook)
+        Pal.logger.info("Beginning execution of playbook ...")
+        ctx = @handler.process_runbook
+
+        log_info "No exporter defined." unless runbook.exporter
+        log_info "No candidates found." unless ctx.candidates.size.positive?
+
+        runbook.exporter.perform_export(ctx)
+      end
+    end
+  end
+end

data/lib/pal/handler/processor.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "pal"
+require "pal/operation/processor_context"
+
+# Processor for CSV extraction
+module Pal
+  module Operation
+    # Base class for CSV impls, we can define strategy on memory usage needs based on
+    # potential issues from file size.
+    # TODO: We probably want to break away from this being a "CSV"-only file type later
+    # Needs more thinking
+    class CSVProcessor
+      include Pal::Log
+
+      # Strategy to return correct type - memory or performance focused.
+      # @return [BaseCSVProcessor]
+      def self.retrieve_default_processor(csv_file_location)
+        Pal.logger.info("Default processor has been requested. No further action required.")
+        RCSVProcessorImpl.new(csv_file_location)
+      end
+
+      attr_accessor :csv_file_location
+
+      def initialize(csv_file_location)
+        @csv_file_location = csv_file_location
+      end
+
+      # @param [ProcessorContext] ctx
+      # @param [Proc] block
+      # @param [Hash] opts
+      def parse(ctx, opts={}, &block)
+        _parse_impl(ctx, opts, &block)
+      end
+
+      private
+
+      # @abstract
+      # @param [ProcessorContext] _ctx
+      # @param [Hash] _opts
+      # @param [Proc] _block
+      def _parse_impl(_ctx, _opts, &_block)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      # @param [String] file_location
+      # @return [String]
+      def read_file(file_location)
+        log_info("Reading file from disk located at #{file_location}")
+        # File.read(File.expand_path(File.join(File.dirname(__FILE__), file_location)))
+        Common::LocalFileUtils.read_file(file_location)
+      end
+
+      # @param [String] _file_location
+      def stream_file(_file_location)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+    end
+
+    require "rcsv"
+
+    # rCSV impl
+    class RCSVProcessorImpl < CSVProcessor
+      private
+
+      # @param [ProcessorContext] ctx
+      # @param [Proc] _block
+      # @param [Hash] opts
+      # @yield [Array] row
+      # @yield [ProcessorContext] ctx
+      # @return [ProcessorContext]
+      def _parse_impl(ctx, opts={}, &_block)
+        return nil unless block_given?
+
+        Rcsv.parse(read_file(@csv_file_location), opts) do |row|
+          ctx.total_row_count += 1
+          ctx.current_file_row_count += 1
+
+          yield row
+        end
+      end
+    end
+  end
+end

data/lib/pal/log.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "pal"
+
+module Pal
+  module Log
+    # @param [String] message
+    def log_debug(message)
+      Pal.logger.debug(message)
+    end
+
+    # @param [String] message
+    def log_info(message)
+      Pal.logger.info(message)
+    end
+
+    # @param [String] message
+    def log_warn(message)
+      Pal.logger.warn(message)
+    end
+
+    # @param [String] message
+    # @param [Exception/Nil] exception
+    def log_error(message, exception=nil)
+      Pal.logger.error(message)
+      Pal.logger.error(exception.backtrace.join("\n")) unless exception.nil?
+    end
+  end
+end

data/lib/pal/main.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "pal"
+require "pal/configuration"
+require "pal/plugin"
+require "pal/handler/base"
+
+module Pal
+  class Main
+    include Log
+    include Plugin
+    include Configuration
+
+    # @return [Pal::Request::Runbook]
+    attr_accessor :runbook
+
+    # @return [Pal::Handler::Manager]
+    attr_accessor :manager
+
+    # @param [Pal::Config] config
+    def initialize(config)
+      register_config(config)
+    end
+
+    # set config for process
+    def setup
+      register_plugins
+
+      @runbook = create_runbook(config.template_file_loc)
+      @manager = create_service_manager
+    end
+
+    # @return [Array, Hash]
+    def process
+      @manager.process_runbook(@runbook)
+    end
+
+    # @param [String] file_location
+    # @return [Pal::Request::Runbook]
+    def create_runbook(file_location)
+      file_relative = file_location.start_with?("/") ? file_location : File.join(File.dirname(__FILE__), file_location)
+
+      log_debug "Attempting to read file from [#{file_relative}]"
+      log_debug "Script executed from [#{__dir__}]"
+
+      request_content = File.read(file_relative)
+      Pal::Request::Runbook.new.from_json(request_content)
+    rescue JSON::ParserError => e
+      log_error("Malformed JSON request for file [#{file_location}]")
+      raise e, "Malformed JSON request for file [#{file_location}]"
+    end
+
+    # @return [Pal::Handler::Manager]
+    def create_service_manager
+      clazz_name = "Pal::Handler::#{@runbook.metadata.handler}HandlerImpl"
+      impl = Kernel.const_get(clazz_name).new(@runbook)
+      Pal::Handler::Manager.new(impl)
+    rescue NameError => e
+      log_error("Cannot find a valid handler impl for #{@runbook.metadata.handler}")
+      raise e
+    end
+  end
+end

data/lib/pal/operation/actions.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require "pal"
+require "pal/operation/projection"
+
+module Pal
+  module Operation
+    class Actions
+      include ObjectHelpers
+      include Log
+
+      # @return [Array<String>]
+      attr_accessor :group_by
+
+      # @return [String]
+      attr_accessor :sort_by
+
+      # @return [Projection]
+      attr_reader :projection
+
+      def projection=(opts)
+        clazz_name = "Pal::Operation::#{opts["type"]&.to_s&.capitalize || "Default"}ProjectionImpl"
+        @projection = Kernel.const_get(clazz_name).new(opts["property"] || nil)
+      end
+
+      def processable?
+        # Do better in the future
+        !@group_by.nil?
+      end
+
+      # @param [Array] rows
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def process(rows, column_headers)
+        grouped = perform_group_by(rows, column_headers)
+
+        return [rows, column_headers] unless @projection&.processable?
+
+        log_info("Performing projection by [#{@projection.type}].")
+        rows, column_headers = perform_projection(grouped, column_headers)
+        rows, column_headers = perform_sort_by(rows, column_headers)
+
+        [rows, column_headers]
+      end
+
+      private
+
+      # @param [Hash] groups
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def perform_projection(groups, column_headers)
+        rows, column_headers = @projection.process(@group_by, groups, column_headers)
+        [rows, column_headers]
+      end
+
+      # @param [Array] rows
+      # @param [Hash] column_headers
+      # @return [Hash] group_by_map
+      def perform_group_by(rows, column_headers)
+        log_info("Performing grouping by #{@group_by} across a total of #{rows.size} has been provided.")
+
+        group_by_map = {}
+        rows.each do |row|
+          key = generate_map_key(row, column_headers)
+          group_by_map[key] = [] unless group_by_map.key?(key)
+
+          group_by_map[key] << row
+        end
+
+        group_by_map
+      end
+
+      # @param [Array] rows
+      # @param [Hash] column_headers
+      # @return [Array] rows, column_headers
+      def perform_sort_by(rows, column_headers)
+        log_info("Performing sort by #{@sort_by} across a total of #{rows.size} has been provided.")
+        return [rows, column_headers] if @sort_by.nil?
+
+        sort_idx = column_headers[@sort_by]
+
+        if sort_idx.nil? || sort_idx.negative?
+          raise "Missing [#{@sort_by}]. Valid candidates are: [#{column_headers.keys.join(", ")}]"
+        end
+
+        rows.sort_by! { |a| a[sort_idx] }
+
+        [rows.reverse, column_headers]
+      end
+
+      # Take a row, extract the props, return a key
+      def generate_map_key(row, column_headers)
+        keys = []
+        @group_by.each do |gbp|
+          idx = column_headers[gbp]
+
+          raise "Missing column index. Please include [#{gbp}] in columns #{column_headers.keys}." unless idx
+
+          keys << row[idx]
+        end
+
+        keys.join(".")
+      end
+    end
+  end
+end