remi 0.2.27 → 0.2.28

data/lib/remi/version.rb

@@ -1,3 +1,3 @@
 module Remi
-  VERSION = '0.2.27'
+  VERSION = '0.2.28'
 end

data/remi.gemspec

@@ -13,7 +13,8 @@ Gem::Specification.new do |s|
   s.description = "Data manipulation and ETL in Ruby"
 
   s.rubyforge_project = "Remi"
-  s.add_runtime_dependency "daru", ["0.1.2"]
+#  Making use of a fork for now
+#  s.add_runtime_dependency "daru", ["0.1.2"]
 
   s.add_runtime_dependency 'bond', ['~> 0.5']
   s.add_runtime_dependency 'docile', ['~> 1.1']

data/spec/metadata_spec.rb

@@ -0,0 +1,62 @@
+=begin
+
+this is probably mostly about data subjects
+
+
+calling fields on the data subject should return only the fields defined, even if there
+are additional fields on the dataframe
+
+dataframe metadata is merged into field metadata, always with a preference for field metadata
+
+metadata propagates through 1:1 STTMs
+
+metadata propagates through intermediate dataframes that are not data subjects
+
+
+    puts '---- ROUTE 1 - direct -----'
+    out_activity.df = Remi::DataFrame.create(:daru, [], order: out_activity.fields.keys)#, index: in_activity.df.index)
+    Remi::SourceToTargetMap.apply(in_activity.df, out_activity.df) do
+#      map source(:activity_id, :student_id) .target(:activity_id, :student_id)
+      map source(:activity_id) .target(:activity_id)
+        .transform(->(v) { "-#{v}-" })
+# enforce types needs to be based on the "fields" for the target
+# I might have to convert any Daru Dataframe to Remi dataframes in the STTM
+
+#        .transform(Remi::Transform[:enforce_types].(on_error: :ignore))
+
+      map source(:student_id) .target(:student_id)
+      map source(:student_dob) .target(:student_dob)
+    end
+
+
+    puts "out_activity.fields: #{out_activity.fields}"
+    puts "out_activity.df metadata: #{out_activity.df.vector_metadata}"
+    out_activity.df = out_activity.df[*(out_activity.fields.keys)]
+    puts "out_activity.fields: #{out_activity.fields}"
+    puts "out_activity.df metadata: #{out_activity.df.vector_metadata}"
+    IRuby.display out_activity.df, type: 'text/html'
+
+
+    puts '---- ROUTE 2 - via work_df -----'
+    work_df = Remi::DataFrame.create(:daru, [], order: out_activity.fields.keys)#, index: in_activity.df.index)
+    Remi::SourceToTargetMap.apply(in_activity.df, work_df) do
+      map source(:activity_id) .target(:activity_id)
+#        .transform(Remi::Transform[:enforce_types].(on_error: :ignore))
+
+      map source(:student_id) .target(:student_id)
+      map source(:student_dob) .target(:student_dob)
+    end
+
+    IRuby.display work_df, type: 'text/html'
+    puts "work_df metadata: #{work_df.vector_metadata}"
+
+    puts "out_activity.fields metadata: #{out_activity.fields}"
+    puts "out_activity.df metadata: #{out_activity.df.vector_metadata}"
+    puts "work_df is a #{work_df.class}"
+    out_activity.df = work_df#[*out_activity.fields.keys]
+    puts "out_activity.fields metadata: #{out_activity.fields}"
+    puts "out_activity.df metadata: #{out_activity.df.vector_metadata}"
+    IRuby.display out_activity.df, type: 'text/html'
+
+
+=end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: remi
 version: !ruby/object:Gem::Version
-  version: 0.2.27
+  version: 0.2.28
 platform: ruby
 authors:
 - Sterling Paramore
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-13 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: daru
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.1.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: bond
   requirement: !ruby/object:Gem::Requirement
@@ -200,6 +186,7 @@ files:
 - features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
+- features/metadata.feature
 - features/parameters.feature
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
@@ -215,6 +202,7 @@ files:
 - jobs/all_jobs_shared.rb
 - jobs/copy_source_job.rb
 - jobs/csv_file_target_job.rb
+- jobs/metadata_job.rb
 - jobs/parameters_job.rb
 - jobs/sample_job.rb
 - jobs/sftp_file_target_job.rb
@@ -229,22 +217,18 @@ files:
 - lib/remi/cucumber.rb
 - lib/remi/cucumber/business_rules.rb
 - lib/remi/cucumber/data_source.rb
-- lib/remi/data_source.rb
-- lib/remi/data_source/csv_file.rb
-- lib/remi/data_source/data_frame.rb
-- lib/remi/data_source/postgres.rb
-- lib/remi/data_source/salesforce.rb
+- lib/remi/data_frame.rb
+- lib/remi/data_frame/daru.rb
 - lib/remi/data_subject.rb
-- lib/remi/data_target.rb
-- lib/remi/data_target/csv_file.rb
-- lib/remi/data_target/data_frame.rb
-- lib/remi/data_target/postgres.rb
-- lib/remi/data_target/salesforce.rb
-- lib/remi/data_target/sftp_file.rb
+- lib/remi/data_subject/csv_file.rb
+- lib/remi/data_subject/data_frame.rb
+- lib/remi/data_subject/postgres.rb
+- lib/remi/data_subject/salesforce.rb
+- lib/remi/data_subject/sftp_file.rb
 - lib/remi/extractor/sftp_file.rb
 - lib/remi/field_symbolizers.rb
+- lib/remi/fields.rb
 - lib/remi/job.rb
-- lib/remi/refinements/daru.rb
 - lib/remi/refinements/symbolizer.rb
 - lib/remi/settings.rb
 - lib/remi/sf_bulk_helper.rb
@@ -253,6 +237,7 @@ files:
 - lib/remi/version.rb
 - remi.gemspec
 - spec/extractor/sftp_file_spec.rb
+- spec/metadata_spec.rb
 - spec/remi_spec.rb
 - workbooks/sample_workbook.ipynb
 - workbooks/workbook_helper.rb
@@ -285,6 +270,7 @@ test_files:
 - features/csv_file_target_job.feature
 - features/examples.feature
 - features/formulas.feature
+- features/metadata.feature
 - features/parameters.feature
 - features/sample_job.feature
 - features/sftp_file_target_job.feature
@@ -297,4 +283,5 @@ test_files:
 - features/transforms/prefix.feature
 - features/transforms/truncate.feature
 - spec/extractor/sftp_file_spec.rb
+- spec/metadata_spec.rb
 - spec/remi_spec.rb

data/lib/remi/data_source.rb

@@ -1,13 +0,0 @@
-module Remi
-  module DataSource
-    include DataSubject
-
-    def extract
-      raise "Extract function undefined for #{self.class.name}"
-    end
-
-    def feild_symbolizer
-      Remi::FieldSymbolizers[:standard]
-    end
-  end
-end

data/lib/remi/data_source/csv_file.rb

@@ -1,101 +0,0 @@
-module Remi
-  module DataSource
-    class CsvFile
-      include DataSource
-
-      using Remi::Refinements::Daru
-
-      def self.default_csv_options
-        CSV::DEFAULT_OPTIONS.merge({
-          headers: true,
-          header_converters: Remi::FieldSymbolizers[:standard],
-          converters: [],
-          col_sep: ',',
-          encoding: 'UTF-8',
-          quote_char: '"'
-        })
-      end
-
-
-      def initialize(fields: {}, extractor:, csv_options: {}, filename_field: nil, logger: Remi::Settings.logger)
-        @fields = fields
-        self.extractor = extractor
-        @csv_options = self.class.default_csv_options.merge(csv_options)
-        @filename_field = filename_field
-        @logger = logger
-      end
-
-      attr_accessor :fields
-      attr_reader   :extractor
-      attr_reader   :csv_options
-
-      def field_symbolizer
-        self.class.default_csv_options[:header_converters]
-      end
-
-      def extract
-        @extracted = Array(@extractor.extract)
-      end
-
-      def extracted
-        @extracted || extract
-      end
-
-      def extractor=(arg)
-        case arg
-        when Extractor::SftpFile, Extractor::LocalFile
-          @extractor = arg
-        when String
-          @extractor = Extractor::LocalFile.new(path: arg)
-        when Regexp
-          raise "Adding regex matching to local files would be easy, not done yet"
-        else
-          raise "Unknown extractor of type #{arg.class}: #{arg}"
-        end
-      end
-
-      # Only going to support single file for now
-      def source_filename
-        raise "Multiple source files detected" if extracted.size > 1
-        @source_filename ||= extracted.first
-      end
-
-      def first_line
-        # Readline assumes \n line endings.  Strip out \r if it is a DOS file.
-        @first_line ||= File.open(source_filename) do |f|
-          f.readline.gsub(/\r/,'')
-        end
-      end
-
-      def headers
-        @headers ||= CSV.open(source_filename, 'r', source_csv_options) { |csv| csv.first }.headers
-      end
-
-      def valid_headers?
-        (fields.keys - headers).empty?
-      end
-
-      def to_dataframe
-        # Assumes that each file has exactly the same structure
-        result_df = nil
-        extracted.each_with_index do |filename, idx|
-          @logger.info "Converting #{filename} to a dataframe"
-          csv_df = Daru::DataFrame.from_csv filename, @csv_options
-
-          csv_df[@filename_field] = Daru::Vector.new([filename] * csv_df.size, index: csv_df.index) if @filename_field
-          if idx == 0
-            result_df = csv_df
-          else
-            result_df = result_df.concat csv_df
-          end
-        end
-
-        result_df
-      end
-
-      def df
-        @dataframe ||= to_dataframe
-      end
-    end
-  end
-end

data/lib/remi/data_source/data_frame.rb

@@ -1,16 +0,0 @@
-module Remi
-  module DataSource
-    class DataFrame
-      include DataSubject
-
-      def initialize(fields: {}, **args)
-        @fields = fields
-      end
-
-      def df
-        @dataframe ||= Daru::DataFrame.new([], order: @fields.keys)
-      end
-
-    end
-  end
-end

data/lib/remi/data_source/postgres.rb

@@ -1,58 +0,0 @@
-module Remi
-  module DataSource
-    class Postgres
-      include DataSource
-
-      def initialize(fields: {}, credentials:, query:, logger: Remi::Settings.logger)
-        @fields = fields
-        @credentials = credentials
-        @query = query
-        @logger = logger
-      end
-
-      attr_accessor :fields
-
-      def extract
-        @logger.info "Executing query #{@query}"
-        @raw_result = connection.exec @query
-      end
-
-      def raw_result
-        @raw_result ||= extract
-      end
-
-      def connection
-        @connection ||= PG.connect(
-          host:     @credentials[:host] || 'localhost',
-          port:     @credentials[:port] || 5432,
-          dbname:   @credentials[:dbname],
-          user:     @credentials[:user] || `whoami`.chomp,
-          password: @credentials[:password],
-          sslmode:  @credentials[:sslmode] || 'allow'
-        )
-      end
-
-
-      def to_dataframe
-        # Performance for larger sets could be improved by using bulk query (via COPY)
-        @logger.info "Converting query to a dataframe"
-
-        hash_array = {}
-        raw_result.each do |row|
-          row.each do |field, value|
-            (hash_array[field_symbolizer.call(field)] ||= []) << value
-          end
-        end
-
-        # After converting to DF, clear the PG results to save memory.
-        raw_result.clear
-
-        Daru::DataFrame.new hash_array, order: hash_array.keys
-      end
-
-      def df
-        @dataframe ||= to_dataframe
-      end
-    end
-  end
-end

data/lib/remi/data_source/salesforce.rb

@@ -1,87 +0,0 @@
-module Remi
-  module DataSource
-    class Salesforce
-      include DataSource
-
-      def initialize(fields: {}, object:, query:, credentials:, api: :bulk, logger: Remi::Settings.logger)
-        @fields = fields
-        @sfo = object
-        @query = query
-        @credentials = credentials
-        @api = api
-        @logger = logger
-      end
-
-      attr_accessor :fields
-      attr_accessor :raw_result
-
-      def field_symbolizer
-        Remi::FieldSymbolizers[:salesforce]
-      end
-
-      def extract
-        @raw_result = sf_bulk.query(@sfo, @query, 10000)
-
-        check_for_errors(@raw_result)
-        @raw_result
-      end
-
-      def check_for_errors(sf_result)
-        sf_result['batches'].each do |batch|
-          raise "Error with batch #{batch['id']} - #{batch['state']}: #{batch['stateMessage']}" unless batch['state'].first == 'Completed'
-        end
-      end
-
-      def raw_result
-        @raw_result ||= extract
-      end
-
-
-
-      def restforce_client
-        @restforce_client ||= begin
-          client = Restforce.new(@credentials)
-
-          #run a dummy query to initiate a connection. Workaround for Bulk API problem
-          # https://github.com/yatish27/salesforce_bulk_api/issues/33
-          client.query('SELECT Id FROM Contact LIMIT 1')
-          client
-        end
-      end
-
-      def sf_bulk
-        @sf_bulk ||= SalesforceBulkApi::Api.new(restforce_client).tap { |o| o.connection.set_status_throttle(5) }
-      end
-
-      def to_dataframe
-        @logger.info "Converting salesforce query results to a dataframe"
-
-        hash_array = {}
-        raw_result['batches'].each do |batch|
-          next unless batch['response']
-
-          batch['response'].each do |record|
-            record.each do |field, value|
-              next if ['xsi:type','type'].include? field
-              (hash_array[field.to_sym] ||= []) << case value.first
-                when Hash
-                  value.first["xsi:nil"] == "true" ? nil : value.first
-                else
-                  value.first
-                end
-            end
-          end
-
-          # delete raw result at end of processing to free memory
-          batch['response'] = nil
-        end
-
-        Daru::DataFrame.new hash_array, order: hash_array.keys
-      end
-
-      def df
-        @dataframe ||= to_dataframe
-      end
-    end
-  end
-end