remi 0.0.1 → 0.2.2

data/lib/remi/extractor/sftp_file.rb

@@ -0,0 +1,84 @@
+module Remi
+  module Extractor
+
+    class LocalFile
+      def initialize(path)
+        @path = path
+      end
+
+      def extract
+        @path
+      end
+    end
+
+    class SftpFile
+
+      class FileNotFoundError < StandardError; end
+
+      def initialize(credentials:, remote_file:, remote_folder: '', local_folder: Settings.work_dir, port: '22', most_recent_only: false, logger: Remi::Settings.logger)
+        @credentials = credentials
+        @remote_file = remote_file
+        @remote_folder = remote_folder
+        @local_folder = local_folder
+        @port = port
+        @most_recent_only = most_recent_only
+        @logger = logger
+      end
+
+      attr_reader :logger
+
+      def extract
+        to_download = @most_recent_only ? Array(most_recent_entry(matching_entries)) : matching_entries
+        raise FileNotFoundError, "File not found: #{@remote_file}" if to_download.size == 0
+        download(to_download)
+      end
+
+      def all_entries(remote_folder = @remote_folder)
+        @all_entries ||= connection { |sftp| sftp.dir.entries(File.join("/", remote_folder)) }
+      end
+
+      def matching_entries(match_name = @remote_file)
+        all_entries.select { |e| match_name.match e.name }
+      end
+
+      def most_recent_entry(entries = matching_entries)
+        entries.sort_by { |e| e.attributes.createtime }.reverse!.first
+      end
+
+      def download(to_download = matching_entries, local_folder: @local_folder, ntry: 3)
+        connection do |sftp|
+          to_download.map do |entry|
+            local_file = File.join(local_folder, entry.name)
+            @logger.info "Downloading #{entry.name} to #{local_file}"
+            retry_download(ntry) { sftp.download!(entry.name, local_file) }
+            local_file
+          end
+        end
+      end
+
+
+      private
+
+      def connection(&block)
+        result = nil
+        Net::SFTP.start(@credentials[:host], @credentials[:username], password: @credentials[:password], port: @port) do |sftp|
+          result = yield sftp
+        end
+        result
+      end
+
+      def retry_download(ntry=2, &block)
+        1.upto(ntry).each do |itry|
+          begin
+            block.call
+          rescue RuntimeError => err
+            raise err unless itry < ntry
+            @logger.error "Download failed with error: #{err.message}"
+            @logger.error "Retry attempt #{itry}/#{ntry-1}"
+            sleep(1)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/remi/field_symbolizers.rb

@@ -0,0 +1,17 @@
+module Remi
+  module FieldSymbolizers
+    def self.[](symbolizer)
+      symbolizers[symbolizer]
+    end
+
+    def self.symbolizers
+      @symbolizers ||= {
+        standard: CSV::HeaderConverters[:symbol],
+        salesforce: lambda { |f|
+          f.encode(CSV::ConverterEncoding).strip.gsub(/\s+/, "_").
+                                           gsub(/\W+/, "").to_sym
+        }
+      }
+    end
+  end
+end

data/lib/remi/job.rb

@@ -0,0 +1,200 @@
+module Remi
+  module Job
+    module JobClassMethods
+      attr_accessor :params
+      attr_accessor :lookups
+      attr_accessor :sources
+      attr_accessor :targets
+      attr_accessor :transforms
+
+      def define_param(key, value)
+        @params ||= {}
+        @params[key] = value
+      end
+
+      def define_lookup(name, type_class, options)
+        @lookups ||= []
+        @lookups << name
+
+        define_method(name) do
+          iv_name = instance_variable_get("@#{name}")
+          return iv_name if iv_name
+
+          if type_class == Hash
+            lookup = options
+          else
+            lookup = type_class.new(options)
+          end
+          instance_variable_set("@#{name}", lookup)
+        end
+      end
+
+      def define_source(name, type_class, **options)
+        @sources ||= []
+        @sources << name
+
+        define_method(name) do
+          iv_name = instance_variable_get("@#{name}")
+          return iv_name if iv_name
+
+          source = type_class.new(options)
+          instance_variable_set("@#{name}", source)
+        end
+      end
+
+      def define_target(name, type_class, **options)
+        @targets ||= []
+        @targets << name
+
+        define_method(name) do
+          iv_name = instance_variable_get("@#{name}")
+          return iv_name if iv_name
+
+          target = type_class.new(options)
+          instance_variable_set("@#{name}", target)
+        end
+      end
+
+      def define_transform(name, sources: [], targets: [], &block)
+        @transforms ||= {}
+        @transforms[name] = { sources: Array(sources), targets: Array(targets) }
+
+        define_method(name) do
+          instance_eval { @logger.info "Running transformation #{__method__}" }
+          instance_eval(&block)
+        end
+      end
+
+      def params
+        @params || {}
+      end
+
+      def lookups
+        @lookups || []
+      end
+
+      def sources
+        @sources || []
+      end
+
+      def targets
+        @targets || []
+      end
+
+      def transforms
+        @transforms || {}
+      end
+
+
+      def work_dir
+        Settings.work_dir
+      end
+
+      def self.extended(receiver)
+      end
+
+      def included(receiver)
+        receiver.extend(JobClassMethods)
+        receiver.params     = self.params.merge(receiver.params)
+        receiver.lookups    = self.lookups + receiver.lookups
+        receiver.sources    = self.sources + receiver.sources
+        receiver.targets    = self.targets + receiver.targets
+        receiver.transforms = self.transforms.merge(receiver.transforms)
+      end
+    end
+
+    def self.included(receiver)
+      receiver.extend(JobClassMethods)
+    end
+
+
+    def params
+      self.class.params
+    end
+
+    def lookups
+      self.class.lookups
+    end
+
+    def sources
+      self.class.sources
+    end
+
+    def targets
+      self.class.targets
+    end
+
+    def transforms
+      self.class.transforms
+    end
+
+
+
+    def initialize(delete_work_dir: true, logger: Settings.logger)
+      @delete_work_dir = delete_work_dir
+      @logger = logger
+      create_work_dir
+    end
+
+    def work_dir
+      self.class.work_dir
+    end
+
+    def finalize
+      delete_work_dir
+    end
+
+    def delete_work_dir
+      if @delete_work_dir && (work_dir.match /^#{Dir.tmpdir}/)
+        @logger.info "Deleting temporary directory #{work_dir}"
+        FileUtils.rm_r work_dir
+      else
+        @logger.debug "Not going to delete working directory #{work_dir}"
+        nil
+      end
+    end
+
+    def create_work_dir
+      @logger.info "Creating working directory #{work_dir}"
+      FileUtils.mkdir_p work_dir
+    end
+
+    # Public: Runs any transforms that use the sources and targets selected.  If
+    # source and target is not specified, then all transforms will be run.
+    # If only the source is specified, then all transforms that use any of the
+    # sources will be run.  Same for specified transforms.
+    #
+    # sources - Array of source names
+    # targets - Array of target names
+    #
+    # Returns an array containing the result of each transform.
+    def run_transforms_using(sources: nil, targets: nil)
+      transforms.map do |t, st|
+        selected_sources = (st[:sources] & Array(sources || st[:sources])).size > 0
+        selected_targets = (st[:targets] & Array(targets || st[:targets])).size > 0
+        self.send(t) if selected_sources && selected_targets
+      end
+    end
+
+    def run_all_transforms
+      transforms.map { |t, st| self.send(t) }
+    end
+
+    def load_all_targets
+      targets.each do |target|
+        @logger.info "Loading target #{target}"
+        self.send(target).tap { |t| t.respond_to?(:load) ? t.load : nil }
+      end
+    end
+
+    # Public: Runs all transforms defined in the job.
+    #
+    # Returns the job instance.
+    def run
+      # Do all of the stuff here
+      run_all_transforms
+      load_all_targets
+      self
+    end
+  end
+end

data/lib/remi/lookup/regex_sieve.rb

@@ -0,0 +1,55 @@
+module Remi
+  module Lookup
+
+    # Public: RegexSieve class.  The RegexSieve functions in a manner similar
+    # a hash.  The regex sieve is initialized with a hash where the keys are
+    # regular expressions and the values can be any valid Ruby object.  The order
+    # of the keys matters.  When the regex sieve is accessed using the array
+    # accessor [], it returns the first matching record.  By default, only
+    # the values are returned, but the key and all matching capture groups
+    # can optionally be returned.
+    #
+    # Examples:
+    #
+    #   r = RegexSieve.new({
+    #     /something/ => 'Something',
+    #     /something else/ => 'This will never get matched because the one above will match first',
+    #     /cool$/ => 'Cool',
+    #     /cool beans/ => 'Really Cool'
+    #   })
+    #
+    #   r['something else'] # => 'Something'
+    #   r['cool beans'] # => 'Really Cool'
+    class RegexSieve
+      def initialize(sieve)
+        @sieve = sieve
+      end
+
+      # Public: Array accessor for Regex Sieve.
+      #
+      # key - The string that will be matched to the keys in the sieve.
+      # opt - By default, only the values in the hash used to initialize the sieve
+      #       will be returned.  However, if you want to return the keys or the
+      #       capture groups then use :regex, :match, or both, respectively.
+      #
+      # Example:
+      #   r['something'] # => 'Something
+      #   r['something', :regex] # => { value: 'Something', regex: /something/ }
+      #   r['sometinng', :match, :regex] # => { value: 'Something', regex: /something/, match: #<MatchData "something"> }
+      def [](key, *opt)
+        opt = opt | [:value]
+
+        regex_match = nil
+        found = @sieve.find do |regex, v|
+          regex_match = regex.match(key)
+        end
+
+        return nil if found.nil?
+        full_result = { value: found[1], regex: found[0], match: regex_match }
+
+        full_result.select! { |k, v| opt.include?(k) }
+        full_result.size > 1 ? full_result : full_result.values.first
+      end
+    end
+  end
+end

data/lib/remi/project/features/examples.feature

@@ -0,0 +1,24 @@
+Feature: This tests the creation of example records.
+
+  Background:
+    Given the job is 'Copy Source'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+  Scenario: Simple example record loads in the source and is directly copied to target.
+
+    Given the following example record for 'Source Data':
+      | MyField    | MyOtherField |
+      | Remilspot  | Niblet       |
+    Then the target field 'MyField' is set to the value "Remilspot"
+    And the target field 'MyOtherField' is set to the value "Niblet"
+
+  Scenario: Handling date formulas in the example data with day units.
+
+    Given the following example record for 'Source Data':
+      | Yesterday   | ThreeDaysFromNow  |
+      | *Yesterday* | *3 days from now* |
+    Then the target field 'Yesterday' is the date 1 day ago
+    And the target field 'ThreeDaysFromNow' is the date 3 days from now

data/lib/remi/project/features/formulas.feature

@@ -0,0 +1,64 @@
+Feature: This tests the creation of example records.
+
+  Background:
+    Given the job is 'Copy Source'
+    And the job source 'Source Data'
+    And the job target 'Target Data'
+    And the source 'Source Data'
+    And the target 'Target Data'
+
+  Scenario: Handling date formulas in the example data with day units.
+
+    Given the following example record for 'Source Data':
+      | Yesterday   | Tomorrow   | OneDayAgo   | SevenDaysAgo  | ThreeDaysFromNow  |
+      | *Yesterday* | *Tomorrow* | *1 day ago* | *7 days ago*  | *3 days from now* |
+    Then the target field 'Yesterday' is the date 1 day ago
+    And the target field 'Tomorrow' is the date 1 day from now
+    And the target field 'OneDayAgo' is the date 1 day ago
+    And the target field 'SevenDaysAgo' is the date 7 days ago
+    And the target field 'ThreeDaysFromNow' is the date 3 days from now
+
+  Scenario: Handling date formulas in the example data with month units.
+
+    Given the following example record for 'Source Data':
+      | LastMonth    | NextMonth    | OneMonthAgo   | SevenMonthsAgo  | ThreeMonthsFromNow  |
+      | *Last Month* | *Next Month* | *1 month ago* | *7 months ago*  | *3 months from now* |
+    Then the target field 'LastMonth' is the date 1 month ago
+    And the target field 'NextMonth' is the date 1 month from now
+    And the target field 'OneMonthAgo' is the date 1 month ago
+    And the target field 'SevenMonthsAgo' is the date 7 months ago
+    And the target field 'ThreeMonthsFromNow' is the date 3 months from now
+
+  Scenario: Handling date formulas in the example data with year units.
+
+    Given the following example record for 'Source Data':
+      | LastYear    | NextYear    | OneYearAgo   | SevenYearsAgo  | ThreeYearsFromNow  |
+      | *Last Year* | *Next Year* | *1 year ago* | *7 years ago*  | *3 years from now* |
+    Then the target field 'LastYear' is the date 1 year ago
+    And the target field 'NextYear' is the date 1 year from now
+    And the target field 'OneYearAgo' is the date 1 year ago
+    And the target field 'SevenYearsAgo' is the date 7 years ago
+    And the target field 'ThreeYearsFromNow' is the date 3 years from now
+
+  Scenario: Handling date formulas in the example data with week units.
+
+    Given the following example record for 'Source Data':
+      | LastWeek    | NextWeek    | OneWeekAgo   | SevenWeeksAgo  | ThreeWeeksFromNow  |
+      | *Last Week* | *Next Week* | *1 week ago* | *7 weeks ago*  | *3 weeks from now* |
+    Then the target field 'LastWeek' is the date 1 week ago
+    And the target field 'NextWeek' is the date 1 week from now
+    And the target field 'OneWeekAgo' is the date 1 week ago
+    And the target field 'SevenWeeksAgo' is the date 7 weeks ago
+    And the target field 'ThreeWeeksFromNow' is the date 3 weeks from now
+
+  Scenario: Handling date formulas when set explicitly in the source.
+
+    Given the following example record for 'Source Data':
+      | SomeDate   |
+      | 2015-10-22 |
+    And the source field 'SomeDate' is set to the value "*Yesterday*"
+    Then the target field 'SomeDate' is the date 1 day ago
+
+    When the source field 'SomeDate' is set to the value "*2 months from now*"
+    Then the target field 'SomeDate' is the date 2 months from now
+    Then the target field 'SomeDate' is populated with "*2 months from now*"