remi 0.2.42 → 0.3.0

data/jobs/sub_transform_example_job.rb

@@ -0,0 +1,43 @@
+require_relative 'all_jobs_shared'
+
+class SharedTransforms < Remi::Job
+  sub_transform :id_prefixer, prefix: 'DEFAULT' do
+    # Declare the required data subjects and data subject fields
+    source :st_source, [:st_source_id]
+    target :st_target, [:st_prefixed_id]
+
+    # Do anything that is allowed in a transform with the provided data subjects and fields
+    Remi::SourceToTargetMap.apply(st_source.df, st_target.df) do
+      map source(:st_source_id) .target(:st_prefixed_id)
+        .transform(Remi::Transform::Prefix.new(params[:prefix]))
+    end
+  end
+end
+
+class SubTransformExampleJob < Remi::Job
+  param(:job_prefix) { nil }
+
+  source :my_source do
+    fields({ :id => {} })
+  end
+
+  target :my_target do
+    fields(
+      {
+        :id => {},
+        :default_id => {}
+      }
+    )
+  end
+
+  transform :main do
+    my_target.df = my_source.df.dup
+    sub_trans_params = job.params[:job_prefix].nil? ? {} : { prefix: job.params[:job_prefix] }
+
+    import SharedTransforms.new.id_prefixer, sub_trans_params do
+      map_source_fields :my_source, :st_source, { :id => :st_source_id }
+      map_target_fields :st_target, :my_target, { :st_prefixed_id => :default_id }
+    end
+
+  end
+end

data/jobs/sub_transform_many_to_many_job.rb

@@ -0,0 +1,46 @@
+require_relative 'all_jobs_shared'
+
+class SharedManyToManyTransforms < Remi::Job
+  sub_transform :unique_values do
+    source :fact, [:id]
+    source :dimension, [:id, :beer, :style]
+    target :unique_beers, [:beer, :count]
+    target :unique_styles, [:style, :count]
+
+    flat_df = fact.df.join(dimension.df, on: [:id], how: :left)
+
+    unique_beers.df = flat_df.group_by([:beer]).size.detach_index
+    unique_beers.df.rename_vectors({ :index => :beer, :values => :count })
+
+    unique_styles.df = flat_df.group_by([:style]).size.detach_index
+    unique_styles.df.rename_vectors({ :index => :style, :values => :count })
+  end
+end
+
+class SubTransformManyToManyJob < Remi::Job
+  source :beer_fact do
+    fields({ :fact_sk => {}, :beer_sk => {} })
+  end
+
+  source :beer_dim do
+    fields({ :beer_sk => {}, :name => {}, :style => {} })
+  end
+
+  target :beer_count do
+    fields({ :name => {}, :count => {} })
+  end
+
+  target :style_count do
+    fields({ :style => {}, :count => {} })
+  end
+
+  transform :main do
+    import SharedManyToManyTransforms.new.unique_values do
+      map_source_fields :beer_fact, :fact, { :beer_sk => :id }
+      map_source_fields :beer_dim, :dimension, { :beer_sk => :id, :name => :beer, :style => :style }
+
+      map_target_fields :unique_beers, :beer_count, { :beer => :name, :count => :count }
+      map_target_fields :unique_styles, :style_count, { :style => :style, :count => :count }
+    end
+  end
+end

data/jobs/transforms/concatenate_job.rb

@@ -1,21 +1,25 @@
 require_relative '../all_jobs_shared'
 
-class ConcatenateJob
-  include AllJobsShared
+class ConcatenateJob < Remi::Job
 
-  define_param :delimiter, ','
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :field1 => {},
-      :field2 => {},
-      :field3 => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:delimiter) { ',' }
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  source :source_data do
+    fields(
+      {
+        :field1 => {},
+        :field2 => {},
+        :field3 => {}
+      }
+    )
+  end
+
+  target :target_data
+
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:field1, :field2, :field3) .target(:result_field)
-        .transform(Remi::Transform::Concatenate.new(params[:delimiter]))
+        .transform(Remi::Transform::Concatenate.new(job.params[:delimiter]))
     end
   end
 end

data/jobs/transforms/data_frame_sieve_job.rb

@@ -1,31 +1,36 @@
 require_relative '../all_jobs_shared'
 
-class DataFrameSieveJob
-  include AllJobsShared
+class DataFrameSieveJob < Remi::Job
 
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :id      => {},
-      :level   => {},
-      :program => {},
-      :contact => {}
-    }
-
-  define_source :sieve, Remi::DataSource::DataFrame,
-    fields: {
-      :level   => {},
-      :program => {},
-      :contact => {},
-      :group   => {}
-    }
+  source :source_data do
+    fields(
+      {
+        :id      => {},
+        :level   => {},
+        :program => {},
+        :contact => {}
+      }
+    )
+  end
 
-  define_target :target_data, Remi::DataTarget::DataFrame
+  source :sieve do
+    fields(
+      {
+        :level   => {},
+        :program => {},
+        :contact => {},
+        :group   => {}
+      }
+    )
+  end
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  target :target_data
 
+  transform :main do
     # Hack to convert example to regex
     sieve.df[:program].recode! { |v| (v || '').match(/\A\/.*\/\Z/) ? /#{v[1...-1]}/ : v }
 
+    target_data.df = source_data.df.dup
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:level, :program, :contact) .target(:group)
         .transform(Remi::Transform::DataFrameSieve.new(sieve.df))

data/jobs/transforms/date_diff_job.rb

@@ -1,24 +1,28 @@
 require_relative '../all_jobs_shared'
 
-class DateDiffJob
-  include AllJobsShared
+class DateDiffJob < Remi::Job
 
-  define_param :measure, :days
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :date1 => { type: :date, format: '%Y-%m-%d' },
-      :date2 => { type: :date, format: '%Y-%m-%d' }
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:measure) { :days }
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  source :source_data do
+    fields(
+      {
+        :date1 => { type: :date, format: '%Y-%m-%d' },
+        :date2 => { type: :date, format: '%Y-%m-%d' }
+      }
+    )
+  end
+
+  target :target_data
+
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:date1, :date2) .target(:difference)
         .transform(->(row) {
           row[:date1] = Date.strptime(row[:date1])
           row[:date2] = Date.strptime(row[:date2])
         })
-        .transform(Remi::Transform::DateDiff.new(params[:measure]))
+        .transform(Remi::Transform::DateDiff.new(job.params[:measure]))
     end
   end
 end

data/jobs/transforms/nvl_job.rb

@@ -1,21 +1,25 @@
 require_relative '../all_jobs_shared'
 
-class NvlJob
-  include AllJobsShared
+class NvlJob < Remi::Job
 
-  define_param :default, ''
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :field1 => {},
-      :field2 => {},
-      :field3 => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:default) { '' }
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  source :source_data do
+    fields(
+      {
+        :field1 => {},
+        :field2 => {},
+        :field3 => {}
+      }
+    )
+  end
+
+  target :target_data
+
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:field1, :field2, :field3) .target(:result_field)
-        .transform(Remi::Transform::Nvl.new(params[:default]))
+        .transform(Remi::Transform::Nvl.new(job.params[:default]))
     end
   end
 end

data/jobs/transforms/parse_date_job.rb

@@ -1,28 +1,31 @@
 require_relative '../all_jobs_shared'
 
-class ParseDateJob
-  include AllJobsShared
+class ParseDateJob < Remi::Job
 
-  define_param :format, '%Y-%m-%d'
-  define_param :if_blank, nil
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :date_string  => { type: :date, in_format: params[:format] },
-      :stubbed_date => { type: :date, in_format: params[:format] }
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:format) { '%Y-%m-%d' }
+  param(:if_blank) { nil }
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  source :source_data do
+    fields(
+      {
+        :date_string  => { type: :date, in_format: params[:format] },
+        :stubbed_date => { type: :date, in_format: params[:format] }
+      }
+    )
+  end
+
+  target :target_data
 
+  transform :main do
     # Only needed for testing, would be nice to make it testable without this
-    params[:if_blank] = ['high', 'low'].include?(params[:if_blank]) ? params[:if_blank].to_sym : params[:if_blank]
+    job.params[:if_blank] = ['high', 'low'].include?(job.params[:if_blank]) ? job.params[:if_blank].to_sym : job.params[:if_blank]
 
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:date_string) .target(:parsed_date)
-        .transform(Remi::Transform::ParseDate.new(in_format: params[:format], if_blank: params[:if_blank]))
+        .transform(Remi::Transform::ParseDate.new(in_format: job.params[:format], if_blank: job.params[:if_blank]))
 
       map source(:stubbed_date) .target(:parsed_stubbed_date)
-        .transform(Remi::Transform::ParseDate.new(in_format: source_data.fields[:stubbed_date][:in_format], if_blank: params[:if_blank]))
+        .transform(Remi::Transform::ParseDate.new(in_format: source_data.fields[:stubbed_date][:in_format], if_blank: job.params[:if_blank]))
     end
   end
 end

data/jobs/transforms/partitioner_job.rb

@@ -1,28 +1,36 @@
 require_relative '../all_jobs_shared'
 
-class PartitionerJob
-  include AllJobsShared
-
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :id      => {}
-    }
+class PartitionerJob < Remi::Job
+
+  source :source_data do
+    fields(
+      {
+        :id      => {}
+      }
+    )
+  end
 
-  define_source :distribution, Remi::DataSource::DataFrame,
-    fields: {
-      :group  => {},
-      :weight => {}
-    }
+  source :distribution do
+    fields(
+      {
+        :group  => {},
+        :weight => {}
+      }
+    )
+  end
 
-  define_source :current_population, Remi::DataSource::DataFrame,
-    fields: {
-      :group => {},
-      :count => {}
-    }
+  source :current_population do
+    fields(
+      {
+        :group => {},
+        :count => {}
+      }
+    )
+  end
 
-  define_target :target_data, Remi::DataTarget::DataFrame
+  target :target_data
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  transform :main do
 
     distribution_hash = distribution.df.map(:row) { |row| [row[:group], row[:weight].to_f] }.to_h
     current_population_hash = current_population.df.map(:row) { |row| [row[:group], row[:count].to_i] }.to_h

data/jobs/transforms/prefix_job.rb

@@ -1,19 +1,22 @@
 require_relative '../all_jobs_shared'
 
-class PrefixJob
-  include AllJobsShared
+class PrefixJob < Remi::Job
 
-  define_param :prefix, 'prefix'
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :my_field => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:prefix) { 'prefix' }
+  source :source_data do
+    fields(
+      {
+        :my_field => {}
+      }
+    )
+  end
+
+  target :target_data
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:my_field) .target(:prefixed_field)
-        .transform(Remi::Transform::Prefix.new(params[:prefix]))
+        .transform(Remi::Transform::Prefix.new(job.params[:prefix]))
     end
   end
 end

data/jobs/transforms/truncate_job.rb

@@ -1,19 +1,23 @@
 require_relative '../all_jobs_shared'
 
-class TruncateJob
-  include AllJobsShared
+class TruncateJob < Remi::Job
 
-  define_param :truncate_len, 5
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :my_field => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  param(:truncate_len) { 5 }
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  source :source_data do
+    fields(
+      {
+        :my_field => {}
+      }
+    )
+  end
+
+  target :target_data
+
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:my_field) .target(:truncated_field)
-        .transform(Remi::Transform::Truncate.new(params[:truncate_len].to_i))
+        .transform(Remi::Transform::Truncate.new(job.params[:truncate_len].to_i))
     end
   end
 end

data/jobs/transforms/truthy_job.rb

@@ -1,15 +1,18 @@
 require_relative '../all_jobs_shared'
 
-class TruthyJob
-  include AllJobsShared
+class TruthyJob < Remi::Job
 
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :truthy => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  source :source_data do
+    fields(
+      {
+        :truthy => {}
+      }
+    )
+  end
+
+  target :target_data
 
-  define_transform :main, sources: :source_data, targets: :target_data do
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map source(:truthy) .target(:allow_nils)
         .transform(Remi::Transform::Truthy.new(allow_nils: true))