cure 0.1.1 → 0.4.0

data/lib/cure/extract/base_processor.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/csv_lookup"
+
+require "csv"
+
+module Cure
+  module Extract
+    class BaseProcessor
+
+      # @return [Cure::DatabaseService]
+      attr_reader :database_service
+
+      # @param [Cure::DatabaseService] database_service
+      def initialize(database_service)
+        @database_service = database_service
+      end
+
+      protected
+
+      # @param [String] tbl_name
+      # @param [Array<Object>] columns
+      def create_table(tbl_name, columns)
+        candidate_column_names = []
+        columns.each_with_index do |col, idx|
+          candidate_column_names << (col || "col_#{idx}")
+        end
+
+        @database_service.create_table(tbl_name.to_sym, candidate_column_names)
+      end
+
+      # @param [String] tbl_name
+      # @param [Array<Object>] values
+      def insert_record(tbl_name, values)
+        @database_service.insert_row(tbl_name.to_sym, values)
+      end
+
+      # @param [String] tbl_name
+      # @param [Array<Array<Object>>] values
+      def insert_batched_rows(tbl_name, values)
+        @database_service.insert_batched_rows(tbl_name.to_sym, values)
+      end
+    end
+  end
+end

data/lib/cure/extract/csv_lookup.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Cure
+  module Extract
+    class CsvLookup
+
+      # @param [String,Integer] position - [Ex A1:B1, A1:B1,A2:B2]
+      # @return [Array] [column_start_idx, column_end_idx, row_start_idx, row_end_idx]
+      def self.array_position_lookup(position)
+        # This is a better way, still trying to figure out a better way but -1 doesn't work for ranges.
+        # return [0, -1, 0, -1] if position.is_a?(Integer) && position == -1
+        return [0, 1_023, 0, 10_000_000] if position.is_a?(Integer) && position == -1 # Whole sheet
+
+        start, finish, *_excess = position.split(":")
+        raise "Invalid format" unless start || finish
+
+        [
+          position_for_letter(start),
+          position_for_letter(finish),
+          position_for_digit(start),
+          position_for_digit(finish)
+        ]
+      end
+
+      # @param [String] range
+      def self.position_for_letter(range)
+        result = 0
+        range.upcase.scan(/[A-Z]+/).first&.each_char do |n|
+          result *= 26
+          result += n.ord - 65 + 1
+        end
+
+        # Excel columns are not 0th indexed.
+        result - 1
+      end
+
+      # @param [String] range
+      def self.position_for_digit(range)
+        range.upcase.scan(/\d+/).first.to_i - 1
+      end
+    end
+  end
+end

data/lib/cure/extract/extractor.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/database"
+require "cure/extract/csv_lookup"
+require "cure/helpers/file_helpers"
+require "cure/helpers/perf_helpers"
+require "cure/extract/named_range_processor"
+require "cure/extract/variable_processor"
+
+require "csv"
+require "objspace"
+
+module Cure
+  module Extract
+    class Extractor
+      include Log
+      include Database
+      include Configuration
+      include Helpers::FileHelpers
+      include Helpers::PerfHelpers
+
+      # @return [Hash] opts
+      attr_reader :opts
+
+      # @param [Hash] opts
+      def initialize(opts)
+        @opts = opts
+      end
+
+      # @param [Pathname,String] file - location of file
+      # @param [String] ref_name - name of reference file
+      def parse_csv(file, ref_name:)
+        nr_processor = named_range_processor(ref_name: ref_name)
+        v_processor = variable_processor(ref_name: ref_name)
+
+        sample_rows = config.template.extraction.sample_rows
+        row_count = 0
+
+        database_service.with_transaction do
+          CSV.foreach(file, liberal_parsing: true) do |row|
+            next if sample_rows && row_count >= sample_rows
+
+            nr_processor.process_row(row_count, row)
+            v_processor.process_row(row_count, row)
+            row_count += 1
+
+            log_info "#{row_count} rows processed [#{Time.now}]" if (row_count % 1_000).zero?
+          end
+
+          nr_processor.after_process
+        end
+
+        log_info "[#{row_count}] total rows parsed from CSV"
+      end
+
+      private
+
+      # @param [String] ref_name - name of reference file
+      # @return [Cure::Extract::NamedRangeProcessor]
+      def named_range_processor(ref_name:)
+        candidate_nrs = config.template.extraction.required_named_ranges(ref_name: ref_name)
+
+        if candidate_nrs.empty?
+          candidate_nrs = [NamedRange.default_named_range(name: ref_name)]
+        end
+
+        NamedRangeProcessor.new(database_service, candidate_nrs)
+      end
+
+      # @param [String] ref_name - name of reference file
+      # @return [Cure::Extract::VariableProcessor]
+      def variable_processor(ref_name:)
+        variables = config.template.extraction.required_variables(ref_name: ref_name)
+        VariableProcessor.new(database_service, variables || [])
+      end
+    end
+  end
+end

data/lib/cure/extract/filter.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Cure
+  module Extract
+    class Filter
+
+      # @return [Filter::RowHandler] row_handler
+      attr_reader :row_handler
+
+      # @return [Filter::ColumnHandler] col_handler
+      attr_reader :col_handler
+
+      def initialize
+        @row_handler = RowHandler.new
+        @col_handler = ColumnHandler.new
+      end
+
+      def columns(&block)
+        return unless block
+
+        @col_handler.instance_eval(&block)
+      end
+
+      def rows(&block)
+        return unless block
+
+        @row_handler.instance_eval(&block)
+      end
+
+      class ColumnHandler
+
+        attr_reader :definitions, :source_col_positions
+
+        def initialize
+          @definitions = []
+          @source_col_positions = nil
+        end
+
+        # @param [String] source
+        # @param [String] as
+        def with(source:, as: nil)
+          @definitions << {
+            source: source,
+            as: as || source
+          }
+
+          self
+        end
+
+        # @param [Array<String>] columns_arr
+        def set_col_positions(columns_arr)
+          @source_col_positions = @definitions.each_with_object({}) do |d, hash|
+            hash[columns_arr.index(d[:source])] = d
+          end
+        end
+
+        # @param [Array<String>] columns_arr
+        def translate_headers(columns_arr)
+          return columns_arr unless has_content?
+
+          @source_col_positions.map do |position, val|
+            if position.nil?
+              raise "Cannot find header position for #{val[:source]}. Please check it exists."
+            end
+
+            columns_arr[position] = val[:as]
+          end
+        end
+
+        # @param [Array<String>] columns_arr
+        def filter_row(columns_arr)
+          return columns_arr unless has_content?
+
+          @source_col_positions.keys.map {|k| columns_arr[k] }
+        end
+
+        # @return [TrueClass, FalseClass]
+        def has_content?
+          @definitions.any?
+        end
+      end
+
+      class RowHandler
+
+        attr_accessor :start_proc, :finish_proc, :including_proc
+
+        # @param [String] where
+        # @param [Hash] options
+        def start(where:, options: {})
+          @start_proc = {where:, options:}
+
+          self
+        end
+
+        # @param [String] where
+        # @param [Hash] options
+        def finish(where:, options: {})
+          @finish_proc = {where:, options:}
+
+          self
+        end
+
+        # @param [String] where
+        # @param [Hash] options
+        def including(where:, options: {})
+          @including_proc = {where:, options:}
+
+          self
+        end
+
+        # @return [TrueClass, FalseClass]
+        def has_content?
+          !!(@start_proc || @finish_proc || @including_proc)
+        end
+      end
+    end
+  end
+end

data/lib/cure/extract/named_range.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "cure/extract/filter"
+
+module Cure
+  module Extract
+    class NamedRange
+
+      def self.default_named_range(name: nil)
+        name ||= "_default"
+
+        new(name, -1)
+      end
+
+      attr_accessor :filter, :row_count
+      attr_reader :name, :section, :headers, :ref_name, :placeholder
+
+      # This is complex purely to support headers not being the 0th row.
+      # A template can specify that the headers row be completely disconnected
+      # from the content, thus we have three bounds:
+      # - Content bounds
+      # - Header bounds
+      # - Sheet bounds (headers AND content)
+
+      # @param [String] ref_name - file reference (for multiple files)
+      def initialize(name, section, headers: nil, ref_name: nil, placeholder: false)
+        @name = name
+        @filter = Filter.new
+        @section = Extract::CsvLookup.array_position_lookup(section)
+        @headers = calculate_headers(headers)
+        @row_count = 0
+        @placeholder = placeholder
+        @ref_name = ref_name || "_default"
+      end
+
+      # @param [Integer] row_idx
+      # @return [TrueClass, FalseClass]
+      def row_in_bounds?(row_idx)
+        row_bounds_range.cover?(row_idx)
+      end
+
+      # @param [Integer] row_idx
+      # @return [TrueClass, FalseClass]
+      def header_in_bounds?(row_idx)
+        header_bounds_range.cover?(row_idx)
+      end
+
+      # @param [Integer] row_idx
+      # @return [TrueClass, FalseClass]
+      def content_in_bounds?(row_idx)
+        content_bounds_range.cover?(row_idx)
+      end
+
+      # @return [Range]
+      def row_bounds_range
+        @row_bounds_range ||= (row_bounds&.first..row_bounds&.last)
+      end
+
+      def row_bounds
+        @row_bounds ||= content_bounds.concat(header_bounds).uniq.minmax
+      end
+
+      # @return [Range]
+      def content_bounds_range
+        @content_bounds_range ||= (content_bounds[0]..content_bounds[1])
+      end
+
+      def content_bounds
+        @content_bounds ||= @section[2..3]
+      end
+
+      # @return [Range]
+      def header_bounds_range
+        @header_bounds_range ||= (header_bounds&.first..header_bounds&.last)
+      end
+
+      def header_bounds
+        @header_bounds ||= @headers[2..3]
+      end
+
+      def active_row_count(row_idx)
+        row_idx - @row_count
+      end
+
+      private
+
+      def calculate_headers(headers)
+        return Extract::CsvLookup.array_position_lookup(headers) if headers
+
+        [@section[0], @section[1], @section[2], @section[2]]
+      end
+    end
+  end
+end

data/lib/cure/extract/named_range_processor.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/csv_lookup"
+require "cure/extract/base_processor"
+
+require "csv"
+
+module Cure
+  module Extract
+    class NamedRangeProcessor < BaseProcessor
+
+      # @return [Array<Extraction::NamedRange>] named_ranges
+      attr_reader :candidate_nrs
+
+      def initialize(database_service, candidate_nrs)
+        @candidate_nrs = candidate_nrs
+        @cache = init_cache
+
+        @tables_created = []
+        super database_service
+      end
+
+      # @param [Integer] row_idx
+      # @param [Array] csv_row
+      def process_row(row_idx, csv_row) # rubocop:disable all
+        # Return if row is not in any named range
+        return unless row_bounds.cover?(row_idx)
+
+        # Iterate over the NR's, if its inside those bounds, add it
+        @candidate_nrs.each do |nr|
+          next unless nr.row_in_bounds?(row_idx)
+
+          if nr.filter.row_handler.has_content?
+            unless nr.filter.row_handler.including_proc[:where].call(csv_row)
+              nr.row_count += 1
+              next
+            end
+          end
+
+          # Row is inbounds - we need to do two things, filter the content, create the table, insert the row
+          if nr.header_in_bounds?(nr.active_row_count(row_idx))
+            column_headers = csv_row[nr.section[0]..nr.section[1]]
+
+            if nr.filter.col_handler.has_content?
+              nr.filter.col_handler.set_col_positions(column_headers)
+              column_headers = nr.filter.col_handler.translate_headers(column_headers)
+            end
+
+            # Create table, flush cache
+            create_table(nr.name, column_headers)
+            @tables_created << nr.name
+
+            @cache[nr.name].each do |row|
+              insert_record(
+                nr.name,
+                nr.filter.col_handler.filter_row(row)
+              )
+            end
+
+            @cache[nr.name] = [] # Evict cache
+
+            next
+          end
+
+          next unless nr.content_in_bounds?(row_idx)
+
+          # 0. Remove unnecessary columns
+
+
+          # 2. If cache is over n records and if the table exists,
+          # add it to the database.
+
+          filtered_row = nr.filter.col_handler.filter_row(
+            csv_row[nr.section[0]..nr.section[1]]
+          )
+
+          if @tables_created.include?(nr.name)
+            @cache[nr.name] << filtered_row.unshift(row_idx)
+
+            if @cache[nr.name].size >= 10
+              insert_cache(nr.name)
+              next
+            end
+          else
+            # If the table doesnt exist, cache it for now.
+            @cache[nr.name] << filtered_row.unshift(row_idx)
+          end
+        end
+      end
+
+      def after_process
+        @cache.each do |named_range, cache|
+          insert_cache(named_range) if cache.size.positive?
+        end
+      end
+
+      # @return [Range]
+      # This covers the max size of all named ranges
+      def row_bounds
+        @row_bounds ||= calculate_row_bounds
+      end
+
+      # @return [Range]
+      def calculate_row_bounds
+        positions = @candidate_nrs.map(&:row_bounds).flatten.sort
+        (positions.first..positions.last)
+      end
+
+      private
+
+      def init_cache
+        cache = {}
+        @candidate_nrs.each do |nr|
+          cache[nr.name] = []
+        end
+
+        cache
+      end
+
+      def insert_cache(named_range)
+        insert_batched_rows(named_range, @cache[named_range])
+        @cache[named_range] = []
+      end
+    end
+  end
+end

data/lib/cure/extract/variable.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Cure
+  module Extract
+    class Variable
+      attr_reader :name, :location, :ref_name
+
+      def initialize(name, location, ref_name: "_default")
+        @name = name
+        @location = [Extract::CsvLookup.position_for_letter(location),
+                     Extract::CsvLookup.position_for_digit(location)]
+        @ref_name = ref_name
+      end
+
+      def row_in_bounds?(row_idx)
+        row_bounds_range.cover?(row_idx)
+      end
+
+      # @return [Range]
+      def row_bounds_range
+        @row_bounds_range ||= (@location&.last..@location&.last)
+      end
+    end
+  end
+end

data/lib/cure/extract/variable_processor.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/base_processor"
+
+require "csv"
+
+module Cure
+  module Extract
+    class VariableProcessor < BaseProcessor
+
+      # @return [Array<Extraction::Variable>] variables
+      attr_reader :candidate_variables
+
+      def initialize(database_service, candidate_variables)
+        super(database_service)
+
+        @candidate_variables = candidate_variables
+        @candidate_count = candidate_variables.length
+        @processed = 0
+
+        init_db
+      end
+
+      # @param [Integer] row_idx
+      # @param [Array] csv_row
+      def process_row(row_idx, csv_row)
+        # Return if row is not in any variable OR if all candidates are processed.
+
+        return if @candidate_count == @processed
+        return unless candidate_rows.include?(row_idx)
+
+        # Iterate over the NR's, if its inside those bounds, add it
+        @candidate_variables.each do |cv|
+          next unless cv.row_in_bounds?(row_idx)
+
+          insert_record(:variables, [nil, cv.name, csv_row[cv.location.first]])
+          @processed += 1
+        end
+      end
+
+      # @return [Array]
+      def candidate_rows
+        @candidate_rows ||= @candidate_variables.map { |v| v.location.last }
+      end
+
+      private
+
+      def init_db
+        return if @database_service.table_exist?(:variables)
+
+        create_table(:variables, %w[name value])
+      end
+    end
+  end
+end

data/lib/cure/generator/base_generator.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Cure
+  module Generator
+    class BaseGenerator
+      # @return [Hash]
+      attr_accessor :options
+
+      def initialize(options={})
+        @options = options
+      end
+
+      # @param [Object/Nil] source_value
+      # @param [RowCtx/Nil] row_ctx
+      # @return [String]
+      def generate(source_value, row_ctx)
+        translated = _generate(source_value, row_ctx)
+        translated = "#{prefix}#{translated}" if prefix
+        translated = "#{translated}#{suffix}" if suffix
+        translated
+      end
+
+      def describe
+        _describe
+      end
+
+      private
+
+      # @param [RowCtx] _row_ctx
+      # @param [Object/Nil] _source_value
+      # @return [String]
+      def _generate(_source_value, _row_ctx)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def _describe
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def prefix(default=nil)
+        extract_property("prefix", default)
+      end
+
+      def suffix(default=nil)
+        extract_property("suffix", default)
+      end
+
+      def length(default=nil)
+        extract_property("length", default)
+      end
+
+      def property_name(default=nil)
+        extract_property("name", default)
+      end
+
+      def extract_property(property, default_val)
+        @options.fetch(property.to_sym, @options.fetch(property, default_val))
+      end
+    end
+  end
+end

data/lib/cure/generator/case_generator.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class CaseGenerator < BaseGenerator
+      private
+
+      # @param [Object] source_value
+      # @param [RowCtx] _row_ctx
+      def _generate(source_value, _row_ctx)
+        result = case_options.fetch(:switch)
+                             .find { |opts| opts[:case] == source_value }
+                             &.fetch(:return_value, nil)
+
+        return result if result
+
+        case_options.fetch(:else, {}).fetch(:return_value, nil)
+      end
+
+      # @return [Hash]
+      def case_options
+        @case_options ||= extract_property("statement", nil)
+      end
+
+      def _describe
+        "Will match source value against a value included in #{case_options}"
+      end
+    end
+  end
+end