remi 0.3.0 → 0.3.1

data/lib/remi.rb

@@ -4,6 +4,8 @@ File.expand_path(File.dirname(__FILE__)).tap {|pwd| $LOAD_PATH.unshift(pwd) unle
 require 'yaml'
 require 'json'
 require 'tmpdir'
+require 'fileutils'
+
 
 # Gems
 require 'daru'
@@ -56,10 +58,11 @@ require 'remi/loader'
 require 'remi/data_subject'
 require 'remi/data_subjects/file_system'
 require 'remi/data_subjects/local_file'
+#require 'remi/data_subjects/gsheet' # intentionally not included by default (must be optionally added)
 require 'remi/data_subjects/sftp_file'
 require 'remi/data_subjects/s3_file'
 require 'remi/data_subjects/csv_file'
-#require 'remi/data_subjects/salesforce' # intentionally not included by default
+#require 'remi/data_subjects/salesforce' # intentionally not included by default (must be optionally added)
 require 'remi/data_subjects/postgres'
 require 'remi/data_subjects/data_frame'
 require 'remi/data_subjects/none'

data/lib/remi/data_subject.rb

@@ -64,6 +64,7 @@ module Remi
 
     # @return [Remi::DataFrame] the dataframe associated with this DataSubject
     def df
+      dsl_eval
       @dataframe ||= Remi::DataFrame.create(df_type, [], order: fields.keys)
     end
 
@@ -71,6 +72,7 @@ module Remi
     # @param new_dataframe [Object] The new dataframe object to be associated.
     # @return [Remi::DataFrame] the associated dataframe
     def df=(new_dataframe)
+      dsl_eval
       if new_dataframe.respond_to? :df_type
         @dataframe = new_dataframe
       else
@@ -259,6 +261,7 @@ module Remi
     # @param obj [Object] adds a loader object to the list of loaders
     # @return [Array] the full list of loaders
     def loader(obj)
+      obj.context = self
       loaders << obj unless loaders.include? obj
     end
 
@@ -269,7 +272,7 @@ module Remi
     # @return [true] if successful
     def load
       return nil if @loaded || df.size == 0
-      dsl_eval if @block
+      dsl_eval
 
       load!
       @loaded = true
@@ -284,6 +287,12 @@ module Remi
       true
     end
 
+    def df=(new_dataframe)
+      super
+      loaders.each { |l| l.load encoded_dataframe if l.autoload }
+      df
+    end
+
     private
 
     # @return [Object] the encoded data suitable for the loaders

data/lib/remi/data_subjects/file_system.rb

@@ -50,6 +50,8 @@ module Remi
     attr_reader :most_recent_only
     attr_reader :group_by
     attr_reader :most_recent_by
+    attr_reader :created_within
+
 
     # Public: Called to extract files from the source filesystem.
     #
@@ -70,6 +72,8 @@ module Remi
         most_recent_matching_entry_in_group
       elsif @most_recent_only
         Array(most_recent_matching_entry)
+      elsif @created_within
+        get_created_within
       else
         matching_entries
       end
@@ -83,6 +87,29 @@ module Remi
       matching_entries.sort_by { |e| e.send(@most_recent_by) }.reverse.first
     end
 
+    def get_created_within
+
+      if @most_recent_only
+        first_entry = matching_entries.sort_by { |e| e.send(@most_recent_by)}.reverse.first
+        if ((Date.today.to_time - Time.at(first_entry.create_time)) / 1.hour) < @created_within
+          Array(first_entry)
+        else
+          raise "No file Found. All files are older than #{@created_within} hrs"
+        end
+      else
+        entries_with_group = matching_entries.map do |entry|
+          if ((Time.new.to_time - Time.at(entry.create_time) ) / 1.seconds) < @created_within
+            entry
+          end
+        end.compact
+        if entries_with_group.length > 0
+          Array(entries_with_group)
+        else
+          raise "No files Found. All files are older than #{@created_within} hrs"
+        end
+      end
+    end
+
     def most_recent_matching_entry_in_group
       entries_with_group = matching_entries.map do |entry|
         match = entry.name.match(@group_by)
@@ -103,13 +130,16 @@ module Remi
 
     private
 
-    def init_file_system(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, **kargs, &block)
+    def init_file_system(*args, remote_path:, pattern: /.*/, local_path: Settings.work_dir, most_recent_only: false, group_by: nil, most_recent_by: :create_time, created_within: nil, **kargs, &block)
+
       @remote_path = Pathname.new(remote_path)
       @pattern = pattern
       @local_path = Pathname.new(local_path)
       @most_recent_only = most_recent_only
       @group_by = group_by
       @most_recent_by = most_recent_by
+      @created_within = created_within
+
     end
   end
 end

data/lib/remi/data_subjects/gsheet.rb

@@ -0,0 +1,140 @@
+require 'google/apis/sheets_v4'
+require 'google/apis/drive_v3'
+require 'googleauth'
+require 'googleauth/stores/file_token_store'
+require 'googleauth/user_refresh'
+
+module Remi
+
+  # Contains methods shared between Salesforce Extractor/Parser/Encoder/Loader
+  class Extractor::Gsheet < Extractor::FileSystem
+
+    def initialize(*args, **kargs, &block)
+      super
+      init_gsheet_extractor(*args, **kargs)
+    end
+
+    attr_reader :data
+    attr_reader :client_id
+    attr_reader :client_secret
+    attr_reader :access_token
+    attr_reader :ref_token
+    attr_reader :scope
+    attr_reader :expire_time
+
+    def authorize
+      credentials = Google::Auth::UserRefreshCredentials.new(
+        client_id:     @client_id,
+        client_secret: @client_secret,
+        scope:         @scope,
+        access_token:  @access_token,
+        refresh_token: @refresh_token,
+        expires_at:    @expiration_time / 1000
+      )
+    end
+
+
+    def get_file_list(folder_id)
+      service                                 = Google::Apis::DriveV3::DriveService.new
+      service.client_options.application_name = @application_name
+      service.authorization                   = authorize()
+      response                                = service_list_files(service, folder_id)
+      response.files
+    end
+
+    def service_list_files(service, folder_id)
+      service.list_files(q: "'#{folder_id}' in parents", page_size: 10, order_by: 'createdTime desc', fields: 'nextPageToken, files(id, name, createdTime, mimeType)')
+    end
+
+    def get_spreadsheet_vals(service, spreadsheet_id)
+      service.get_spreadsheet_values(spreadsheet_id, 'Sheet1')
+    end
+
+    def extract
+      service                                 = Google::Apis::SheetsV4::SheetsService.new
+      service.client_options.application_name = @application_name
+      service.authorization                   = authorize()
+      @data                                   = []
+
+      entries.each do |file|
+        response = get_spreadsheet_vals(service, file.raw)
+        data.push(response)
+      end
+
+      self
+    end
+
+    # @return [Array<Extractor::FileSystemEntry>] (Memoized) list of objects in the bucket/prefix
+    def all_entries
+      @all_entries ||= all_entries!
+    end
+
+    # @return [Array<Extractor::FileSystemEntry>] (Memoized) list of objects in the bucket/prefix
+    def all_entries!
+      gsheet_entries = get_file_list(@default_folder_id)
+      gsheet_entries.map do |entry|
+        entry = entry.to_h
+        FileSystemEntry.new(
+          pathname:       File.join(@default_folder_id, entry[:name]),
+          create_time:    entry[:created_time],
+          modified_time:  entry[:created_time],
+          raw:            entry[:id]
+        )
+      end
+    end
+
+    private
+
+    def init_gsheet_extractor(*args, credentials:, folder_id:, **kargs)
+      @default_folder_id   = folder_id
+      @oob_uri             = 'urn:ietf:wg:oauth:2.0:oob'
+      @application_name    = credentials.fetch(:application_name)
+
+      @client_secrets_path = File.join(
+        Dir.home,
+        '.credentials/client_secret.json'
+      )
+      @credentials_path = File.join(
+        Dir.home,
+        '.credentials/sheets.googleapis.com-ruby-remi.yaml'
+      )
+      @client_id       = credentials.fetch(:client_id)
+      @access_token    = credentials.fetch(:access_token)
+      @refresh_token   = credentials.fetch(:refresh_token)
+      @client_secret   = credentials.fetch(:client_secret)
+      @project_id      = credentials.fetch(:project_id)
+      @scope           = ["https://www.googleapis.com/auth/drive","https://www.googleapis.com/auth/spreadsheets"]
+      @expiration_time = Integer(credentials.fetch(:expiration_time))
+    end
+  end
+
+  class Parser::Gsheet < Parser
+
+    def parse(gs_extract)
+      google_vals = gs_extract.data
+      return_hash = nil
+      google_vals.each do |google_val|
+
+        if return_hash.nil?
+          return_hash = Hash.new
+          google_val.values[0].each do |header|
+            return_hash[field_symbolizer.call(header)] = []
+          end
+        end
+
+        keys_temp = return_hash.keys
+
+        google_val.values[1..-1].each do |rows|
+          col_num = 0
+
+          rows.each do |value|
+            return_hash[keys_temp[col_num]] << value
+            col_num +=1
+          end
+        end
+      end
+      Remi::DataFrame.create(:daru, return_hash, order: return_hash.keys)
+    end
+  end
+
+end

data/lib/remi/data_subjects/sftp_file.rb

@@ -61,6 +61,7 @@ module Remi
           logger.info "Downloading #{entry.name} to #{local_file}"
           retry_download { sftp.download!(File.join(@remote_path, entry.name), local_file) }
           local_file
+
         end
       end
     end

data/lib/remi/data_subjects/sub_job.rb

@@ -11,7 +11,8 @@ module Remi
     attr_accessor :sub_job, :data_subject
 
     def extract
-      sub_job.job.send(data_subject).df
+      sub_job.execute unless sub_job.sub_job.send(data_subject).is_a? Remi::DataSource
+      sub_job.sub_job.send(data_subject).df
     end
 
     private
@@ -26,25 +27,33 @@ module Remi
   class Loader::SubJob < Loader
     # @param sub_job [Object] The name (relative to parent job) of the subjob to use
     # @param data_subject [Symbol] The name (relatvie to the sub job) of the sub job's data frame
+    # @param merge_fields [True,False] Indicates whether fields from the calling data subject
+    #   should be merged with those defined in the sub job.
     def initialize(*args, **kargs, &block)
       super
       init_sub_job_loader(*args, **kargs, &block)
     end
 
-    attr_accessor :sub_job, :data_subject
+    attr_accessor :sub_job, :data_subject, :merge_fields
 
     # @param data_frame [Object] Data frame to load to target sub job data subject
     # @return [true] On success
     def load(data_frame)
-      sub_job.job.send(data_subject).df = data_frame
+      sub_job.sub_job.send(data_subject).df = data_frame
+      sub_job.sub_job.send(data_subject).fields.merge! fields if merge_fields
+      true
+    end
+
+    def autoload
       true
     end
 
     private
 
-    def init_sub_job_loader(*args, sub_job:, data_subject:, **kargs, &block)
+    def init_sub_job_loader(*args, sub_job:, data_subject:, merge_fields: true, **kargs, &block)
       @sub_job = sub_job
       @data_subject = data_subject
+      @merge_fields = merge_fields
     end
   end
 end

data/lib/remi/encoder.rb

@@ -38,7 +38,7 @@ module Remi
 
     # @return [Remi::Fields] The fields (uses the context fields if defined)
     def fields
-      return context.fields if context if context.respond_to? :fields
+      return context.fields if context && context.respond_to?(:fields)
       @fields
     end
   end

data/lib/remi/job.rb

@@ -271,7 +271,8 @@ module Remi
         "  parameters: #{params.to_h.keys}\n" +
         "  sources: #{sources}\n" +
         "  targets: #{targets}\n" +
-        "  transforms: #{transforms}"
+        "  transforms: #{transforms}\n" +
+        "  sub_jobs: #{sub_jobs}"
     end
 
 
@@ -282,6 +283,7 @@ module Remi
     # @return [self]
     def execute(*components)
       execute_transforms if components.empty? || components.include?(:transforms)
+      execute_sub_jobs if components.empty? || components.include?(:sub_jobs)
       execute_load_targets if components.empty? || components.include?(:load_targets)
       self
     end
@@ -334,6 +336,12 @@ module Remi
       self
     end
 
+    # Executes all subjobs (not already executed)
+    def execute_sub_jobs
+      sub_jobs.each { |sj| send(sj).execute }
+      self
+    end
+
     # Adds all parameters listed to the job parameters
     def add_params(**kargs)
       kargs.each { |k,v| params[k] = v }

data/lib/remi/job/parameters.rb

@@ -39,6 +39,7 @@ module Remi
     class Parameters
       def initialize(context=nil)
         @context = context
+        @params_methods = []
         @params = {}
       end
 
@@ -65,10 +66,13 @@ module Remi
       def []=(name, value)
         __define__(name) { value } unless respond_to? name
         @params[name] = value
+
+        value
       end
 
-      # @return [Hash] The parameters as a hash
+      # @return [Hash] The evaluated parameters as a hash
       def to_h
+        @params_methods.each { |p| self.send(p) }
         @params
       end
 
@@ -76,13 +80,14 @@ module Remi
       def clone
         the_clone = super
         the_clone.instance_variable_set(:@params, @params.dup)
+        the_clone.instance_variable_set(:@params_methods, @params_methods.dup)
         the_clone
       end
 
       def __define__(name, &block)
-        @params[name] = nil
+        @params_methods << name unless @params_methods.include? name
         define_singleton_method name do
-          @params[name] ||= Remi::Dsl.dsl_return(self, @context, &block)
+          @params.fetch(name) { |name| @params[name] = Remi::Dsl.dsl_return(self, @context, &block) }
         end
       end
     end

data/lib/remi/job/sub_job.rb

@@ -10,25 +10,31 @@ module Remi
       attr_accessor :context, :name
 
       def dsl_return
-        sub_job = Dsl.dsl_return(self, @context, &@block)
-        raise ArgumentError, "SubJob DSL must return a Remi::Job" unless sub_job.is_a? Job
-        sub_job
+        result = Dsl.dsl_return(self, @context, &@block)
+        raise ArgumentError, "SubJob DSL must return a Remi::Job" unless result.is_a? Job
+        result
       end
 
-      def job
-        @job ||= dsl_return
+      def sub_job
+        @sub_job ||= dsl_return
       end
 
       def fields(data_subject)
-        job.send(data_subject).dsl_eval.fields
+        sub_job.send(data_subject).dsl_eval.fields
       end
 
       def execute
-        job.execute
+        execute! unless @executed
+      end
+
+      def execute!
+        result = sub_job.execute
+        @executed = true
+        result
       end
 
       def execute_transforms
-        job.execute(:transforms)
+        sub_job.execute(:transforms)
       end
     end
   end