cure 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4991d1e0e8328be48a27ab4ed1dc099cc57f0b0ebd1ae1add6763fca4908d9c3
-  data.tar.gz: 477a17c2ba7052657d54facea23056f9ad07068b6e3596810da4ae7cfb79d51a
+  metadata.gz: 29e7c915346b45c1407208f6ba1810ece68004c8d7dd9ffb8eb2b5dcca903b35
+  data.tar.gz: c356c649bc4a902fcb347e9e16f2299f4c8a7c0db576a7fc8242d080b575920d
 SHA512:
-  metadata.gz: ee5600adcd1052493ef5db828cc43830e5d758dfb153d849afddb1f3e7a209606036cc238d425081d3d9724ed4cb3d9ee7466dd49df6b80c37711ab25aa53172
-  data.tar.gz: d49ccb85c6bfc5414b666f49395683ef2a751ff7af19c2f380af94aebdda835de8c8cd8bc4dbca17c426d91cb64a2254b35456b204e7327352ba8de97a232747
+  metadata.gz: 5999a52c084656af0a6a18ba68884aec3de825be9f84f5c67cdf771031af414789aaebf24dfc9fed52f6a76b9a9495081f0f4e1c0f872927fbe34c271fa86e62
+  data.tar.gz: 458962e9905380ef0e5832ef9e2ae2da1ede47f448992314bceba95c997c2af3bb4d6f2b92e1f23e906563c77ad1fbcd37d171d75732cf29c15b6b9b0e05e5bc

data/.rubocop.yml

@@ -26,6 +26,9 @@ Metrics/ClassLength:
 Layout/SpaceAroundEqualsInParameterDefault:
   EnforcedStyle: no_space
 
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
 # We do not need to support Ruby 1.9, so this is good to use.
 Style/SymbolArray:
   Enabled: true

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cure (0.1.1)
+    cure (0.1.2)
       faker
       rcsv
 

data/README.md

@@ -1,19 +1,29 @@
 # Cure
 
 ![run tests](https://github.com/williamthom-as/cure/actions/workflows/rspec.yml/badge.svg)
+[![Gem Version](https://badge.fury.io/rb/cure.svg)](https://badge.fury.io/rb/cure)
 
-Cure is a simple tool to **remove/redact/anonymize** and **replace** private information in a spreadsheet.
-It has been written to anonymize private cloud billing data for use in public demo environments.
+Cure is a simple tool to **extract/clean/transform/remove/redact/anonymize** and **replace** information in a spreadsheet.
+It has been written to anonymize private cloud billing data for use in public demo environments.  Since then, it has grown to 
+additional processing capabilities that can take a CSV from junk to workable data.
 
 It has several key features:
-- Define either full or regex match groups replacements.
-- Choose from many strategies to replace anonymous data - random number sequences, GUIDs, placeholders, multipliers amongst many others.
-- **Existing generated values are stored and recalled** so once a replacement is defined, it is kept around for other columns to use.
-  - For example, once a replacement **Account Number** is generated, any further use of that number sequence is other columns will be used, keeping data real(ish) and functional in a relational sense.
+- Operate on your data to build what you need. 
+  - Files are taken through an `Extract -> Build -> Transform -> Export` pipeline.
+- Extract parts of your file into named ranges to remove junk. 
+- Build (Add/Remove/Explode) columns - handy for files that may have JSON as a column value.
+- Transform values:
+  - Define either full or regex match groups replacements.
+  - Choose from many strategies to replace anonymous data - random number sequences, GUIDs, placeholders, multipliers amongst many others.
+  - **Existing generated values are stored and recalled** so once a replacement is defined, it is kept around for other columns to use.
+    - For example, once a replacement **Account Number** is generated, any further use of that number sequence is other columns will be used, keeping data real(ish) and functional in a relational sense.
+- Export into one (or many) files, in a selection of chosen formats (CSV at the moment, coming soon with JSON, Parquet).
 
 ## Use Cases
 
 - Strip out personal data from a CSV that may be used for public demo.
+- Extract specific parts of a CSV file and junk the rest.
+- Explode JSON values into individual columns per key.
 
 ## Usage
 

data/exe/cure

@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
+# rubocop:disable Style/MixinUsage
+# rubocop:disable Style/MethodCalledOnDoEndBlock
+
 require "cure"
 require "optparse"
 
@@ -34,9 +37,10 @@ OptionParser.new do |opts|
 end.parse!
 
 log_info "Config loaded successfully, initialising environment ..."
-main = Cure::Main.init(conf[:template_file_location], conf[:source_file_location], conf[:output_dir])
+main = Cure::Main.init_from_file(conf[:template_file_location], conf[:source_file_location], conf[:output_dir])
 
 log_info "... set up complete. Beginning process"
-main.process
-
+main.run_export
 
+# rubocop:enable Style/MixinUsage
+# rubocop:enable Style/MethodCalledOnDoEndBlock

data/lib/cure/config.rb

@@ -19,16 +19,30 @@ module Cure
       ConfigurationSource.instance.load_config(request_config)
     end
 
+    # @param [String] source_file_location
+    # @param [Cure::Template] template
+    # @param [String] output_dir
     # @return [Config]
     def create_config(source_file_location, template, output_dir)
       Config.new(source_file_location, template, output_dir)
     end
 
+    # If we are overloading here as a "data store" and "config store", we
+    # could break out variables and placeholders into their own singleton.
+    #
+    # This should be a kind of instance cache, which loads once per run,
+    # and junk can be jammed in there?
     class Config
-      attr_accessor :source_file_location, :template, :output_dir
+      attr_accessor :source_file_location, :output_dir
+
+      # @return [Cure::Template] template
+      attr_accessor :template
+
+      # @return [Hash] variables
+      attr_accessor :variables
 
       # @param [String] source_file_location
-      # @param [Hash] template
+      # @param [Cure::Template] template
       # @param [String] output_dir
       def initialize(source_file_location, template, output_dir)
         @source_file_location = source_file_location
@@ -37,7 +51,7 @@ module Cure
       end
 
       def placeholders
-        @template["placeholders"] || []
+        @template.transformations.placeholders || {}
       end
     end
 

data/lib/cure/coordinator.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/helpers/file_helpers"
+
+require "cure/extract/extractor"
+require "cure/transformation/transform"
+require "cure/export/exporter"
+
+require "rcsv"
+
+module Cure
+  # Coordinates the entire process:
+  # Extract -> Build -> Transform -> Export
+  class Coordinator
+    include Configuration
+    include Log
+
+    def process
+      # need to check config is init'd
+
+      extracted_csv = extract
+      built_csv = build(extracted_csv)
+      transformed_csv = transform(built_csv)
+      export(transformed_csv)
+    end
+
+    private
+
+    # @return [Cure::Extract::WrappedCSV]
+    def extract
+      log_info "Beginning the extraction process..."
+
+      extractor = Extract::Extractor.new({})
+      result = extractor.extract_from_file(config.source_file_location)
+
+      log_debug "Setting extracted variables to global conf for access downstream"
+      config.variables = result.variables
+
+      log_info "...extraction complete"
+      result
+    end
+
+    # @param [Cure::Extract::WrappedCSV] wrapped_csv
+    def build(wrapped_csv)
+      log_info "Beginning the building process..."
+      log_info "... building complete"
+      wrapped_csv
+    end
+
+    # @return [Hash<String,Cure::Transformation::TransformResult>]
+    # @param [Cure::Extract::WrappedCSV] parsed_csv
+    def transform(parsed_csv)
+      log_info "Beginning the transformation process..."
+      transformer = Cure::Transformation::Transform.new(config.template.transformations.candidates)
+      content = transformer.transform_content(parsed_csv)
+
+      log_info "...transform complete"
+      content
+    end
+
+    # @return [Hash<String,Cure::Transformation::TransformResult>]
+    def export(transformed_result)
+      log_info "Beginning export process..."
+      Cure::Export::Exporter.export_result(transformed_result, config.output_dir)
+      log_info "... export complete."
+
+      transformed_result
+    end
+  end
+end

data/lib/cure/export/exporter.rb

@@ -1,24 +1,49 @@
 # frozen_string_literal: true
 
 require "cure/log"
-require "cure/file_helpers"
+require "cure/config"
+require "cure/helpers/file_helpers"
 
 module Cure
   module Export
     class Exporter
-      include Cure::FileHelpers
+      include Helpers::FileHelpers
+      include Configuration
       include Log
 
-      def self.export_ctx(ctx, output_dir, file_name)
-        column_headers = ctx.column_headers.keys
+      # @param [Array<Cure::Transform::TransformResult>] result
+      def self.export_result(results, output_dir)
+        exporter = Exporter.new(output_dir)
+        exporter.export_results(results)
+      end
+
+      attr_reader :output_dir
+
+      def initialize(output_dir)
+        @output_dir = output_dir
+      end
 
-        exporter = Exporter.new
-        exporter.export(output_dir, file_name, ctx.transformed_rows, column_headers)
+      # @param [Array<Cure::Transform::TransformResult>] result
+      def export_results(result)
+        export_ranges = config.template.dispatch.named_ranges
+
+        export_ranges.each do |range|
+          named_range = range["named_range"]
+          unless result.has_key?(named_range)
+            raise "Missing named range - #{range} from candidates [#{result.keys.join(", ")}]"
+          end
+
+          data = result[named_range]
+          column_headers = data.column_headers.keys
+          export(@output_dir, range["file_name"], data.transformed_rows, column_headers)
+        end
       end
 
       # @param [Array] rows
       # @param [Array] columns
       def export(output_dir, file_name, rows, columns)
+        file_name = "#{file_name}-#{Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S%-z")}"
+
         log_info("Exporting file to [#{output_dir}/#{file_name}] with #{rows.length} rows")
 
         file_contents = []
@@ -37,7 +62,7 @@ module Cure
       # @param [String] contents
       # @param [String] file_extension
       def write_to_file(file_path, file_name, file_extension, contents)
-        file_location = "#{file_path}/#{file_name || Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S%-z")}"
+        file_location = "#{file_path}/#{file_name}"
         clean_dir(file_path)
 
         with_file(file_location, file_extension) do |file|

data/lib/cure/extract/builder.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "cure/extract/csv_lookup"
+
+module Cure
+  module Extract
+    class Builder
+
+      # @param [Hash] opts
+      attr_reader :opts
+
+      # @param [Hash] opts
+      def initialize(opts)
+        @opts = opts
+      end
+
+      # @param [Array<Array>] _sheet
+      # @param [Hash<String, Integer>] _column_headers
+      # @return [Array]
+      #
+      # This returns changed column headers and sheets
+      def handle(_sheet, _column_headers)
+        []
+      end
+    end
+  end
+end

data/lib/cure/extract/csv_lookup.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Cure
+  module Extract
+    class CsvLookup
+
+      # @param [String] position - [Ex A1:B1, A1:B1,A2:B2]
+      # @return [Array] [column_start_idx, column_end_idx, row_start_idx, row_end_idx]
+      def self.array_position_lookup(position)
+        return [0, -1, 0, -1] if position.is_a?(Integer) && position == -1 # Whole sheet
+
+        start, finish, *_excess = position.split(":")
+        raise "Invalid format" unless start || finish
+
+        [
+          position_for_letter(start),
+          position_for_letter(finish),
+          position_for_digit(start),
+          position_for_digit(finish)
+        ]
+      end
+
+      def self.position_for_letter(range)
+        range.upcase.scan(/[A-Z]+/).first.ord - 65 # A (65) - 65 = 0 idx
+      end
+
+      def self.position_for_digit(range)
+        range.upcase.scan(/\d+/).first.to_i - 1
+      end
+    end
+  end
+end

data/lib/cure/extract/extractor.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "cure/log"
+require "cure/config"
+require "cure/extract/csv_lookup"
+require "cure/helpers/file_helpers"
+
+module Cure
+  module Extract
+    class Extractor
+      include Log
+      include Configuration
+      include Helpers::FileHelpers
+
+      # @param [Hash] opts
+      attr_reader :opts
+
+      # @param [Hash] opts
+      def initialize(opts)
+        @opts = opts
+      end
+
+      # @param [String] csv_file_location
+      # @return [WrappedCSV]
+      def extract_from_file(csv_file_location)
+        file_contents = read_file(csv_file_location)
+        extract_from_contents(file_contents)
+      end
+
+      # @param [String] file_contents
+      # @return [WrappedCSV]
+      def extract_from_contents(file_contents)
+        parsed_content = parse_csv(file_contents, header: :none)
+        log_info("Parsed CSV into #{parsed_content.content.length} sections.")
+        parsed_content
+      end
+
+      # private
+
+      # @param [String] file_contents
+      # @param [Hash] opts
+      # @return [WrappedCSV]
+      def parse_csv(file_contents, opts={})
+        csv_rows = []
+
+        Rcsv.parse(file_contents, opts) { |row| csv_rows << row }
+
+        result = WrappedCSV.new
+        result.content = extract_named_ranges(csv_rows)
+        result.variables = extract_variables(csv_rows)
+
+        result
+      end
+
+      # @param [Array<Array>] csv_rows
+      # @return [Array<Hash>]
+      def extract_named_ranges(csv_rows)
+        # Use only the NR's that are defined from the candidates list
+        candidates = config.template.transformations.candidates
+        candidate_nrs = config.template.extraction.required_named_ranges(candidates.map(&:named_range).uniq)
+        candidate_nrs.map do |nr|
+          {
+            "rows" => extract_from_rows(csv_rows, nr["section"]),
+            "name" => nr["name"]
+          }
+        end
+      end
+
+      # @param [Array<Array>] csv_rows
+      # @return [Hash]
+      def extract_variables(csv_rows)
+        config.template.extraction.variables.each_with_object({}) do |variable, hash|
+          hash[variable["name"]] = lookup_location(csv_rows, variable["location"])
+        end
+      end
+
+      # @param [Array<Array>] rows
+      def extract_from_rows(rows, named_range)
+        psx = CsvLookup.array_position_lookup(named_range)
+
+        ret_val = []
+        rows.each_with_index do |row, idx|
+          # If the position of the end row is -1, we need all,
+          # otherwise if its between/equal to start/finish
+          ret_val << row[psx[0]..psx[1]] if psx[3] == -1 || (idx >= psx[2] && idx <= psx[3])
+        end
+
+        ret_val
+      end
+
+      # @param [Array<Array>] rows
+      # @param [String] variable_location
+      def lookup_location(rows, variable_location)
+        psx = [CsvLookup.position_for_letter(variable_location),
+               CsvLookup.position_for_digit(variable_location)]
+        rows[psx[1]][psx[0]]
+      end
+
+      # @param [Integer] row_idx
+      # @param [Array] row
+      # @param [Array] psx
+      # @return [Array, nil]
+      def handle_row(row_idx, row, psx)
+        return nil unless psx[3] == -1 || (row_idx >= psx[2] && row_idx <= psx[3])
+
+        row[psx[0]..psx[1]]
+      end
+    end
+
+    class WrappedCSV
+      # @return [Array<Hash>]
+      attr_accessor :content
+
+      # @return [Hash]
+      attr_accessor :variables
+
+      def initialize
+        @content = []
+        @variables = {}
+      end
+    end
+  end
+end

data/lib/cure/generator/base_generator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Cure
+  module Generator
+    class BaseGenerator
+      # @return [Hash]
+      attr_accessor :options
+
+      def initialize(options={})
+        @options = options
+      end
+
+      # @param [Object/Nil] source_value
+      # @return [String]
+      def generate(source_value=nil)
+        translated = _generate(source_value)
+        translated = "#{prefix}#{translated}" if prefix
+        translated = "#{translated}#{suffix}" if suffix
+        translated
+      end
+
+      private
+
+      # @param [Object/Nil] _source_value
+      # @return [String]
+      def _generate(_source_value)
+        raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
+      end
+
+      def prefix(default=nil)
+        extract_property("prefix", default)
+      end
+
+      def suffix(default=nil)
+        extract_property("suffix", default)
+      end
+
+      def length(default=nil)
+        extract_property("length", default)
+      end
+
+      def property_name(default=nil)
+        extract_property("name", default)
+      end
+
+      def extract_property(property, default_val)
+        @options.fetch(property, default_val)
+      end
+    end
+  end
+end

data/lib/cure/generator/case_generator.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class CaseGenerator < BaseGenerator
+      private
+
+      # @param [Object] source_value
+      def _generate(source_value)
+        result = case_options.fetch("switch").find { |opts| opts["case"] == source_value }&.fetch("return_value", nil)
+
+        return result if result
+
+        case_options.fetch("else", {}).fetch("return_value", nil)
+      end
+
+      # @return [Hash]
+      def case_options
+        @case_options ||= extract_property("statement", nil)
+      end
+    end
+  end
+end

data/lib/cure/generator/character_generator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class CharacterGenerator < BaseGenerator
+
+      def initialize(options=nil)
+        super(options)
+      end
+
+      private
+
+      # @param [Object] source_value
+      def _generate(source_value)
+        arr = build_options.map(&:to_a).flatten
+        (0...length(source_value&.length || 5)).map { arr[rand(arr.length)] }.join
+      end
+
+      def build_options
+        return [("a".."z"), ("A".."Z"), (0..9)] unless @options.key?("types")
+
+        type_array = @options["types"]
+
+        arr = []
+        arr << ("a".."z") if type_array.include? "lowercase"
+        arr << ("A".."Z") if type_array.include? "uppercase"
+        arr << (0..9) if type_array.include? "number"
+        arr << ("!".."+") if type_array.include? "symbol"
+        arr
+      end
+    end
+  end
+end

data/lib/cure/generator/faker_generator.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+require "faker"
+
+module Cure
+  module Generator
+    class FakerGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      def _generate(_source_value)
+        mod_code = extract_property("module", nil)
+        mod = Faker.const_get(mod_code)
+
+        raise "No Faker module found for [#{mod_code}]" unless mod
+
+        meth_code = extract_property("method", nil)&.to_sym
+        raise "No Faker module found for [#{meth_code}]" unless mod.methods.include?(meth_code)
+
+        mod.send(meth_code)
+      end
+    end
+  end
+end

data/lib/cure/generator/guid_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class GuidGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      def _generate(_source_value)
+        SecureRandom.uuid.to_s
+      end
+    end
+  end
+end

data/lib/cure/generator/hex_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class HexGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      def _generate(_source_value)
+        1.upto(length(rand(0..9))).map { rand(0..15).to_s(16) }.join("")
+      end
+    end
+  end
+end

data/lib/cure/generator/imports.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+require "cure/generator/hex_generator"
+require "cure/generator/case_generator"
+require "cure/generator/character_generator"
+require "cure/generator/faker_generator"
+require "cure/generator/guid_generator"
+require "cure/generator/number_generator"
+require "cure/generator/placeholder_generator"
+require "cure/generator/redact_generator"
+require "cure/generator/variable_generator"

data/lib/cure/generator/number_generator.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "cure/generator/base_generator"
+
+module Cure
+  module Generator
+    class NumberGenerator < BaseGenerator
+      private
+
+      # @param [Object] _source_value
+      def _generate(_source_value)
+        1.upto(length(rand(0..9))).map { rand(1..9) }.join("").to_i
+      end
+    end
+  end
+end