cure 0.1.2 → 0.4.0

data/lib/cure/extract/named_range_processor.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/csv_lookup"
+require "cure/extract/base_processor"
+
+require "csv"
+
+module Cure
+  module Extract
+    class NamedRangeProcessor < BaseProcessor
+
+      # @return [Array<Extraction::NamedRange>] named_ranges
+      attr_reader :candidate_nrs
+
+      def initialize(database_service, candidate_nrs)
+        @candidate_nrs = candidate_nrs
+        @cache = init_cache
+
+        @tables_created = []
+        super database_service
+      end
+
+      # @param [Integer] row_idx
+      # @param [Array] csv_row
+      def process_row(row_idx, csv_row) # rubocop:disable all
+        # Return if row is not in any named range
+        return unless row_bounds.cover?(row_idx)
+
+        # Iterate over the NR's, if its inside those bounds, add it
+        @candidate_nrs.each do |nr|
+          next unless nr.row_in_bounds?(row_idx)
+
+          if nr.filter.row_handler.has_content?
+            unless nr.filter.row_handler.including_proc[:where].call(csv_row)
+              nr.row_count += 1
+              next
+            end
+          end
+
+          # Row is inbounds - we need to do two things, filter the content, create the table, insert the row
+          if nr.header_in_bounds?(nr.active_row_count(row_idx))
+            column_headers = csv_row[nr.section[0]..nr.section[1]]
+
+            if nr.filter.col_handler.has_content?
+              nr.filter.col_handler.set_col_positions(column_headers)
+              column_headers = nr.filter.col_handler.translate_headers(column_headers)
+            end
+
+            # Create table, flush cache
+            create_table(nr.name, column_headers)
+            @tables_created << nr.name
+
+            @cache[nr.name].each do |row|
+              insert_record(
+                nr.name,
+                nr.filter.col_handler.filter_row(row)
+              )
+            end
+
+            @cache[nr.name] = [] # Evict cache
+
+            next
+          end
+
+          next unless nr.content_in_bounds?(row_idx)
+
+          # 0. Remove unnecessary columns
+
+
+          # 2. If cache is over n records and if the table exists,
+          # add it to the database.
+
+          filtered_row = nr.filter.col_handler.filter_row(
+            csv_row[nr.section[0]..nr.section[1]]
+          )
+
+          if @tables_created.include?(nr.name)
+            @cache[nr.name] << filtered_row.unshift(row_idx)
+
+            if @cache[nr.name].size >= 10
+              insert_cache(nr.name)
+              next
+            end
+          else
+            # If the table doesnt exist, cache it for now.
+            @cache[nr.name] << filtered_row.unshift(row_idx)
+          end
+        end
+      end
+
+      def after_process
+        @cache.each do |named_range, cache|
+          insert_cache(named_range) if cache.size.positive?
+        end
+      end
+
+      # @return [Range]
+      # This covers the max size of all named ranges
+      def row_bounds
+        @row_bounds ||= calculate_row_bounds
+      end
+
+      # @return [Range]
+      def calculate_row_bounds
+        positions = @candidate_nrs.map(&:row_bounds).flatten.sort
+        (positions.first..positions.last)
+      end
+
+      private
+
+      def init_cache
+        cache = {}
+        @candidate_nrs.each do |nr|
+          cache[nr.name] = []
+        end
+
+        cache
+      end
+
+      def insert_cache(named_range)
+        insert_batched_rows(named_range, @cache[named_range])
+        @cache[named_range] = []
+      end
+    end
+  end
+end

data/lib/cure/extract/variable.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Cure
+  module Extract
+    class Variable
+      attr_reader :name, :location, :ref_name
+
+      def initialize(name, location, ref_name: "_default")
+        @name = name
+        @location = [Extract::CsvLookup.position_for_letter(location),
+                     Extract::CsvLookup.position_for_digit(location)]
+        @ref_name = ref_name
+      end
+
+      def row_in_bounds?(row_idx)
+        row_bounds_range.cover?(row_idx)
+      end
+
+      # @return [Range]
+      def row_bounds_range
+        @row_bounds_range ||= (@location&.last..@location&.last)
+      end
+    end
+  end
+end

data/lib/cure/extract/variable_processor.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/base_processor"
+
+require "csv"
+
+module Cure
+  module Extract
+    class VariableProcessor < BaseProcessor
+
+      # @return [Array<Extraction::Variable>] variables
+      attr_reader :candidate_variables
+
+      def initialize(database_service, candidate_variables)
+        super(database_service)
+
+        @candidate_variables = candidate_variables
+        @candidate_count = candidate_variables.length
+        @processed = 0
+
+        init_db
+      end
+
+      # @param [Integer] row_idx
+      # @param [Array] csv_row
+      def process_row(row_idx, csv_row)
+        # Return if row is not in any variable OR if all candidates are processed.
+
+        return if @candidate_count == @processed
+        return unless candidate_rows.include?(row_idx)
+
+        # Iterate over the NR's, if its inside those bounds, add it
+        @candidate_variables.each do |cv|
+          next unless cv.row_in_bounds?(row_idx)
+
+          insert_record(:variables, [nil, cv.name, csv_row[cv.location.first]])
+          @processed += 1
+        end
+      end
+
+      # @return [Array]
+      def candidate_rows
+        @candidate_rows ||= @candidate_variables.map { |v| v.location.last }
+      end
+
+      private
+
+      def init_db
+        return if @database_service.table_exist?(:variables)
+
+        create_table(:variables, %w[name value])
+      end
+    end
+  end
+end

data/lib/cure/generator/base_generator.rb

@@ -11,19 +11,29 @@ module Cure
       end
 
       # @param [Object/Nil] source_value
+      # @param [RowCtx/Nil] row_ctx
       # @return [String]
-      def generate(source_value=nil)
-        translated = _generate(source_value)
+      def generate(source_value, row_ctx)
+        translated = _generate(source_value, row_ctx)
         translated = "#{prefix}#{translated}" if prefix
         translated = "#{translated}#{suffix}" if suffix
         translated
       end
 
+      def describe
+        _describe
+      end
+
       private
 
+      # @param [RowCtx] _row_ctx
       # @param [Object/Nil] _source_value
       # @return [String]
-      def _generate(_source_value)
+      def _generate(_source_value, _row_ctx)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def _describe
         raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
       end
 
@@ -44,7 +54,7 @@ module Cure
       end
 
       def extract_property(property, default_val)
-        @options.fetch(property, default_val)
+        @options.fetch(property.to_sym, @options.fetch(property, default_val))
       end
     end
   end

data/lib/cure/generator/case_generator.rb

@@ -8,18 +8,25 @@ module Cure
       private
 
       # @param [Object] source_value
-      def _generate(source_value)
-        result = case_options.fetch("switch").find { |opts| opts["case"] == source_value }&.fetch("return_value", nil)
+      # @param [RowCtx] _row_ctx
+      def _generate(source_value, _row_ctx)
+        result = case_options.fetch(:switch)
+                             .find { |opts| opts[:case] == source_value }
+                             &.fetch(:return_value, nil)
 
         return result if result
 
-        case_options.fetch("else", {}).fetch("return_value", nil)
+        case_options.fetch(:else, {}).fetch(:return_value, nil)
       end
 
       # @return [Hash]
       def case_options
         @case_options ||= extract_property("statement", nil)
       end
+
+      def _describe
+        "Will match source value against a value included in #{case_options}"
+      end
     end
   end
 end

data/lib/cure/generator/character_generator.rb

@@ -13,15 +13,16 @@ module Cure
       private
 
       # @param [Object] source_value
-      def _generate(source_value)
+      # @param [RowCtx] _row_ctx
+      def _generate(source_value, _row_ctx)
         arr = build_options.map(&:to_a).flatten
         (0...length(source_value&.length || 5)).map { arr[rand(arr.length)] }.join
       end
 
       def build_options
-        return [("a".."z"), ("A".."Z"), (0..9)] unless @options.key?("types")
+        return [("a".."z"), ("A".."Z"), (0..9)] unless @options.key?(:types)
 
-        type_array = @options["types"]
+        type_array = @options[:types]
 
         arr = []
         arr << ("a".."z") if type_array.include? "lowercase"
@@ -30,6 +31,11 @@ module Cure
         arr << ("!".."+") if type_array.include? "symbol"
         arr
       end
+
+      def _describe
+        "Will create a random list of #{@options[:types]} " \
+        "with as many characters as the source string."
+      end
     end
   end
 end

data/lib/cure/generator/erb_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+require "erb"
+
+module Cure
+  module Generator
+    class ErbGenerator < BaseGenerator
+      private
+
+      # @param [Cure::Transformation::RowCtx] _source_value
+      # @param [RowCtx] row_ctx
+      def _generate(_source_value, row_ctx)
+        template = @options.fetch(:template, nil)
+        ERB.new(template).result_with_hash(row_ctx.row)
+      end
+
+      def _describe; end
+    end
+  end
+end

data/lib/cure/generator/eval_generator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class EvalGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      # @param [Transformation::RowCtx] _row_ctx
+      # This will be changed with expression evaluator
+      def _generate(_source_value, _row_ctx)
+        eval_str = extract_property("eval", nil)
+        result = nil
+        with_safe do
+          result = eval(eval_str) # rubocop:disable Security/Eval
+        end
+
+        result
+      rescue StandardError => e
+        raise "Cannot eval statement #{extract_property("eval", nil)} [#{e.message}]"
+      end
+
+      def with_safe(&_block)
+        $SAFE = 1
+        yield
+        $SAFE = 0
+      end
+
+      def _describe; end
+    end
+  end
+end

data/lib/cure/generator/faker_generator.rb

@@ -9,7 +9,8 @@ module Cure
       private
 
       # @param [Object] _source_value
-      def _generate(_source_value)
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
         mod_code = extract_property("module", nil)
         mod = Faker.const_get(mod_code)
 
@@ -20,6 +21,11 @@ module Cure
 
         mod.send(meth_code)
       end
+
+      def _describe
+        "Will create a Faker value from [#{extract_property("module", nil)}::" \
+        "#{extract_property("method", nil)}]"
+      end
     end
   end
 end

data/lib/cure/generator/guid_generator.rb

@@ -8,8 +8,13 @@ module Cure
       private
 
       # @param [Object] _source_value
-      def _generate(_source_value)
-        SecureRandom.uuid.to_s
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
+        SecureRandom.uuid.to_s.slice(0, @options.fetch(:length, 36))
+      end
+
+      def _describe
+        "Will create a random GUID."
       end
     end
   end

data/lib/cure/generator/hex_generator.rb

@@ -8,9 +8,14 @@ module Cure
       private
 
       # @param [Object] _source_value
-      def _generate(_source_value)
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
         1.upto(length(rand(0..9))).map { rand(0..15).to_s(16) }.join("")
       end
+
+      def _describe
+        "Will create a random list of hex values matching the length of the source string."
+      end
     end
   end
 end

data/lib/cure/generator/imports.rb

@@ -10,3 +10,7 @@ require "cure/generator/number_generator"
 require "cure/generator/placeholder_generator"
 require "cure/generator/redact_generator"
 require "cure/generator/variable_generator"
+require "cure/generator/eval_generator"
+require "cure/generator/erb_generator"
+require "cure/generator/proc_generator"
+require "cure/generator/static_generator"

data/lib/cure/generator/number_generator.rb

@@ -8,9 +8,14 @@ module Cure
       private
 
       # @param [Object] _source_value
-      def _generate(_source_value)
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
         1.upto(length(rand(0..9))).map { rand(1..9) }.join("").to_i
       end
+
+      def _describe
+        "Will create a random list of numbers matching the length of the source string."
+      end
     end
   end
 end

data/lib/cure/generator/placeholder_generator.rb

@@ -11,10 +11,16 @@ module Cure
       private
 
       # @param [Object] _source_value
-      def _generate(_source_value)
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
         value = config.placeholders[property_name]
         value || raise("Missing placeholder value. Available candidates: [#{config.placeholders.keys.join(", ")}]")
       end
+
+      def _describe
+        "Will look up placeholders using '#{property_name}'. " \
+        "[Set as '#{config.placeholders[property_name]}']"
+      end
     end
   end
 end

data/lib/cure/generator/proc_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+require "erb"
+
+module Cure
+  module Generator
+    class ProcGenerator < BaseGenerator
+      private
+
+      # @param [Cure::Transformation::RowCtx] source_value
+      # @param [RowCtx] row_ctx
+      def _generate(source_value, row_ctx)
+        proc = @options.fetch(:execute, nil)
+        proc.call(source_value, row_ctx)
+      end
+
+      def _describe; end
+    end
+  end
+end

data/lib/cure/generator/redact_generator.rb

@@ -7,10 +7,16 @@ module Cure
     class RedactGenerator < BaseGenerator
       private
 
-      # @param [Object] source_value
-      def _generate(source_value)
-        1.upto(length(source_value&.length || 5)).map { "X" }.join("")
+      # @param [object] source_value
+      # @param [RowCtx] _row_ctx
+      def _generate(source_value, _row_ctx)
+        1.upto(length(source_value&.length || 5)).map { @options.fetch(:character, "x") }.join("")
       end
+
+      def _describe
+        "Will replace the length of the source string with X."
+      end
+
     end
   end
 end

data/lib/cure/generator/static_generator.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class StaticGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
+        @options.fetch(:value, nil)
+      end
+
+      def _describe
+        "Will return the defined value [#{@options.fetch(:value, nil)}]"
+      end
+    end
+  end
+end

data/lib/cure/generator/variable_generator.rb

@@ -2,18 +2,24 @@
 
 require "cure/generator/base_generator"
 require "cure/config"
+require "cure/database"
 
 module Cure
   module Generator
     class VariableGenerator < BaseGenerator
-      include Cure::Configuration
+      include Database
 
       private
 
-      # @param [Object] _source_value
-      def _generate(_source_value)
-        value = config.variables[property_name]
-        value || raise("Missing placeholder value. Available candidates: [#{config.variables.keys.join(", ")}]")
+      # @param [object] _source_value
+      # @param [RowCtx] _row_ctx
+      def _generate(_source_value, _row_ctx)
+        value = database_service.find_variable(property_name)
+        value || raise("Missing placeholder value [#{property_name}]. Please check you are defining it correctly.")
+      end
+
+      def _describe
+        "Will look up the variables defined using '#{property_name}'."
       end
     end
   end

data/lib/cure/helpers/file_helpers.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require "pathname"
 
 module Cure
   module Helpers
@@ -21,13 +22,22 @@ module Cure
       end
 
       def read_file(file_location)
-        result = file_location.start_with?("/") ? file_location : File.join(File.dirname(__FILE__), file_location)
+        result = file_location.start_with?("/") ? file_location : Pathname.new(file_location)
+        # result = file_location.start_with?("/") ? file_location : File.join(File.dirname(__FILE__), file_location)
 
-        raise "No file found at [#{file_location}]" unless File.exist? result
+        raise "No file found at [#{file_location}]" unless File.exist? result.to_s
 
         File.read(result)
       end
 
+      def open_file(file_location)
+        result = file_location.start_with?("/") ? file_location : Pathname.new(file_location)
+
+        raise "No file found at [#{file_location}]" unless File.exist? result.to_s
+
+        File.open(result)
+      end
+
       def with_temp_dir(temp_dir, &_block)
         return unless block_given?
 

data/lib/cure/helpers/object_helpers.rb

@@ -6,23 +6,11 @@ require "json"
 module Cure
   module Helpers
     module ObjectHelpers
-      def attributes=(hash)
-        hash.each do |key, value|
-          send("#{key}=", value)
-        rescue NoMethodError
-          Cure.logger.warn("Error deserializing object: No property for #{key}")
-        end
-      end
-
-      def from_json(json)
-        return from_hash(json) if json.is_a?(Hash) # Just a guard in case serialisation is done
-
-        from_hash(JSON.parse(json))
-      end
-
-      def from_hash(hash)
-        self.attributes = hash
-        self
+      def class_exists?(klass_name)
+        klass = Module.const_get(klass_name)
+        klass.is_a?(Class)
+      rescue NameError
+        false
       end
     end
   end

data/lib/cure/helpers/perf_helpers.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "benchmark"
+
+module Cure
+  module Helpers
+    # This module uses some code sourced from here:
+    # https://torrocus.com/blog/different-ways-to-processing-large-csv-file-in-ruby/
+    module PerfHelpers
+      def print_memory_usage(process_name="default")
+        cmd = "ps -o rss= -p #{Process.pid}"
+        before_mem = `#{cmd}`.to_i
+        # before_gc = GC.stat(:total_allocated_objects)
+
+        yield
+        after_mem = `#{cmd}`.to_i
+        # after_gc = GC.stat(:total_allocated_objects)
+
+        log_info "Total Memory Usage [#{process_name}]: #{((after_mem - before_mem) / 1024.0).round(2)} MB"
+        # log_info "Total GC Objects Freed [#{process_name}]: #{after_gc - before_gc}"
+      end
+
+      def print_time_spent(process_name="default", &block)
+        time = Benchmark.realtime(&block)
+        log_info "Total Processing Time [#{process_name}]: #{time.round(2)}"
+      end
+    end
+  end
+end