remi 0.2.42 → 0.3.0

data/jobs/json_job.rb

@@ -1,21 +1,25 @@
 require_relative 'all_jobs_shared'
 
-class JsonJob
-  include AllJobsShared
-
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :json_array => { type: :json },
-      :json_hash  => { type: :json }
-    }
+class JsonJob < Remi::Job
+  source :source_data do
+    fields(
+      {
+        :json_array => { type: :json },
+        :json_hash  => { type: :json }
+      }
+    )
+  end
 
-  define_target :target_data, Remi::DataTarget::DataFrame,
-    fields: {
-      :second_element => {},
-      :name_field     => {}
-    }
+  target :target_data do
+    fields(
+      {
+        :second_element => {},
+        :name_field     => {}
+      }
+    )
+  end
 
-  define_transform :main do
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields) do
       map source(:json_array) .target(:second_element)
         .transform(->(values) { values[1] })

data/jobs/metadata_job.rb

@@ -1,37 +1,42 @@
 require_relative 'all_jobs_shared'
 ENV['TZ'] = 'UTC'
 
-class MetadataJob
-  include AllJobsShared
+class MetadataJob < Remi::Job
+  source :source_data do
+    fields(
+      {
+        :activity_id      => { from: 'in', in: true, cdc_type: 2 },
+        :student_id       => { from: 'in', in: true, type: :string, cdc_type: 2 },
+        :student_dob      => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
+        :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
+        :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
+        :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
+        :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 8, scale: 2, cdc_type: 2 },
+        :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
+        :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
+      }
+    )
+  end
 
-  define_source :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :activity_id      => { from: 'in', in: true, cdc_type: 2 },
-      :student_id       => { from: 'in', in: true, type: :string, cdc_type: 2 },
-      :student_dob      => { from: 'in', in: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d', cdc_type: 2 },
-      :activity_type    => { from: 'in', in: true, type: :string, valid_values: ['A', 'B', 'C'], cdc_type: 2 },
-      :activity_counter => { from: 'in', in: true, type: :integer, cdc_type: 2 },
-      :activity_score   => { from: 'in', in: true, type: :float, cdc_type: 2 },
-      :activity_cost    => { from: 'in', in: true, type: :decimal, precision: 8, scale: 2, cdc_type: 2 },
-      :activity_date    => { from: 'in', in: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S', cdc_type: 2 },
-      :source_filename  => { from: 'in', in: true, type: :string, cdc_type: 1 }
-    }
+  target :target_data do
+    encoder Remi::Encoder::CsvFile.new path: "#{Remi::Settings.work_dir}/target_data.csv"
 
-  define_target :target_data, Remi::DataTarget::CsvFile,
-    path: "#{Remi::Settings.work_dir}/target_data.csv",
-    fields: {
-      :activity_id      => { from: 'out', out: true },
-      :student_id       => { from: 'out', out: true, type: :string },
-      :student_dob      => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
-      :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
-      :activity_counter => { from: 'out', out: true, type: :integer },
-      :activity_score   => { from: 'out', out: true, type: :float },
-      :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 8, scale: 2 },
-      :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
-      :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
-    }
+    fields(
+      {
+        :activity_id      => { from: 'out', out: true },
+        :student_id       => { from: 'out', out: true, type: :string },
+        :student_dob      => { from: 'out', out: true, type: :date, in_format: '%m/%d/%Y', out_format: '%Y-%m-%d' },
+        :activity_type    => { from: 'out', out: true, type: :string, valid_values: ['A', 'B', 'C'] },
+        :activity_counter => { from: 'out', out: true, type: :integer },
+        :activity_score   => { from: 'out', out: true, type: :float },
+        :activity_cost    => { from: 'out', out: true, type: :decimal, precision: 8, scale: 2 },
+        :activity_date    => { from: 'out', out: true, type: :datetime, in_format: '%m/%d/%Y %H:%M:%S', out_format: '%Y-%m-%dT%H:%M:%S' },
+        :source_filename  => { from: 'out', out: true, type: :string, cdc_type: 1 }
+      }
+    )
+  end
 
-  define_transform :main do
+  transform :main do
     source_data.enforce_types
 
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df, source_metadata: source_data.fields, target_metadata: target_data.fields) do

data/jobs/parameters_job.rb

@@ -1,22 +1,25 @@
 require_relative 'all_jobs_shared'
 
-class ParametersJob
-  include AllJobsShared
+class ParametersJob < Remi::Job
+  param(:myparam) {}
+  param(:test_parameter) { "my test parameter value" }
 
-  define_param :test_parameter, "my test parameter value"
+  source :source_data do
+    fields(
+      {
+        :parameter_name => {}
+      }
+    )
+  end
 
-  define_target :source_data, Remi::DataSource::DataFrame,
-    fields: {
-      :parameter_name => {}
-    }
-  define_target :target_data, Remi::DataTarget::DataFrame
+  target :target_data
 
-  define_transform :main do
+  transform :main do
     Remi::SourceToTargetMap.apply(source_data.df, target_data.df) do
       map target(:myparam)
-        .transform(Remi::Transform::Constant.new(params[:myparam]))
+        .transform(Remi::Transform::Constant.new(job.params[:myparam]))
       map source(:parameter_name) .target(:parameter_name)
-        .transform(->(v) { params[v.to_sym] })
+        .transform(->(v) { job.params[v.to_sym] })
     end
   end
 

data/jobs/sample_job.rb

@@ -1,88 +1,118 @@
 # This is an example Remi job that was auto-generated by Remi.
 require_relative 'all_jobs_shared'
-require 'remi/data_subject/salesforce'
-
-class SampleJob
-  include AllJobsShared
-
-  define_source :existing_contacts, Remi::DataSource::Salesforce,
-    object: :Contact,
-    credentials: params[:salesforce_credentials],
-    api: :bulk,
-    fields: {
-      :Id             => {},
-      :External_ID__c => {},
-      :IsActive       => { type: :boolean },
-      :CreatedDate    => { type: :date, in_format: '%Y-%m-%d %H:%M:%S' }
-    },
-    query: <<-EOQ
-      SELECT
-        Id,
-        External_ID__c
-      FROM
-        Contact
-    EOQ
-
-
-  define_source :sample_file, Remi::DataSource::CsvFile,
-    extractor: Remi::Extractor::SftpFile.new(
+require 'remi/data_subjects/salesforce'
+
+class SampleJob < AllJobsShared
+
+  param :program_name_lookup do
+    RegexSieve.new(
+      {
+        /^BIO$/              => "Biology",
+        /^Fake Biology$/     => nil,
+        /(?:B|Microb)iology/ => "Biology",
+        /^CHEM$/             => "Chemistry",
+        /Chemistry/          => "Chemistry",
+        /Physics/            => "Physics"
+      }
+    )
+  end
+
+  source :existing_contacts do
+    extractor Remi::Extractor::Salesforce.new(
+      object: :Contact,
+      credentials: params[:salesforce_credentials],
+      api: :bulk,
+      query: <<-EOQ
+        SELECT
+          Id,
+          External_ID__c,
+          IsActive,
+          CreatedDate
+        FROM
+          Contact
+      EOQ
+    )
+    parser Remi::Parser::Salesforce.new
+
+    field_symbolizer :salesforce
+    fields(
+      {
+        :Id             => {},
+        :External_ID__c => {},
+        :IsActive       => { type: :boolean },
+        :CreatedDate    => { type: :date, in_format: '%Y-%m-%d %H:%M:%S' }
+      }
+    )
+  end
+
+  source :sample_file do
+    extractor Remi::Extractor::SftpFile.new(
       credentials: params[:sftp],
       remote_path: '/',
       pattern: /^SampleFile_(\d+)\.txt/,
       most_recent_only: true
-    ),
-    csv_options: {
-      headers: true,
-      col_sep: ",",
-      encoding: "ISO-8859-1:UTF-8"
-    },
-    fields: {
-      :student_id             => {},
-      :school_id              => {},
-      :school_name            => {},
-      :program                => {},
-      :last_name              => {},
-      :first_name             => {},
-      :current_email          => {},
-      :mailing_address_line_1 => {},
-      :mailing_address_line_2 => {},
-      :mailing_city           => {},
-      :mailing_state          => {},
-      :mailing_postal_code    => {},
-      :birthdate              => { type: :date, in_format: '%m/%d/%Y'},
-      :applied_date           => { type: :date, in_format: '%m/%d/%Y'}
-    }
-
-  define_target :all_contacts, Remi::DataTarget::DataFrame
-
-  define_target :contact_updates, Remi::DataTarget::Salesforce,
-    credentials: params[:salesforce_credentials],
-    object: :Contact,
-    operation: :update,
-    api: :bulk
-
-  define_target :contact_creates, Remi::DataTarget::Salesforce,
-    credentials: params[:salesforce_credentials],
-    object: :Contact,
-    operation: :create,
-    api: :bulk
-
-  define_param :program_name_lookup, RegexSieve.new({
-    /^BIO$/              => "Biology",
-    /^Fake Biology$/     => nil,
-    /(?:B|Microb)iology/ => "Biology",
-    /^CHEM$/             => "Chemistry",
-    /Chemistry/          => "Chemistry",
-    /Physics/            => "Physics"
-  })
-
-  define_transform :map_common_fields, sources: [:sample_file, :existing_contacts], targets: :all_contacts do
+    )
+
+    parser Remi::Parser::CsvFile.new(
+      csv_options: {
+        headers: true,
+        col_sep: ",",
+        encoding: "ISO-8859-1:UTF-8"
+      }
+    )
+
+    fields(
+      {
+        :student_id             => {},
+        :school_id              => {},
+        :school_name            => {},
+        :program                => {},
+        :last_name              => {},
+        :first_name             => {},
+        :current_email          => {},
+        :mailing_address_line_1 => {},
+        :mailing_address_line_2 => {},
+        :mailing_city           => {},
+        :mailing_state          => {},
+        :mailing_postal_code    => {},
+        :birthdate              => { type: :date, in_format: '%m/%d/%Y'},
+        :applied_date           => { type: :date, in_format: '%m/%d/%Y'}
+      }
+    )
+  end
+
+
+  target :all_contacts
 
+  target :contact_updates do
+    encoder Remi::Encoder::Salesforce.new
+    loader Remi::Loader::Salesforce.new(
+      credentials: params[:salesforce_credentials],
+      object: :Contact,
+      operation: :update,
+      api: :bulk
+    )
+    field_symbolizer :salesforce
+  end
+
+  target :contact_creates do
+    encoder Remi::Encoder::Salesforce.new
+    loader Remi::Loader::Salesforce.new(
+      credentials: params[:salesforce_credentials],
+      object: :Contact,
+      operation: :create,
+      api: :bulk
+    )
+    field_symbolizer :salesforce
+  end
+
+
+  transform :map_common_fields do
     # Exclude all source records with an invalid program name
     all_contacts.df = sample_file.df.dup
     Remi::SourceToTargetMap.apply(all_contacts.df) do
       map source(:program) .target(:Major__c)
-        .transform(Remi::Transform::Lookup.new(params[:program_name_lookup]))
+        .transform(Remi::Transform::Lookup.new(job.params[:program_name_lookup]))
     end
     all_contacts.df = all_contacts.df.where(all_contacts.df[:Major__c].not_eq(nil))
 
@@ -102,8 +132,7 @@ class SampleJob
   end
 
 
-  define_transform :map_creates, sources: :all_contacts, targets: :contact_creates do
-
+  transform :map_creates do
     work_contact_creates = all_contacts.df.where(all_contacts.df[:Id].eq(nil))
 
     Remi::SourceToTargetMap.apply(work_contact_creates) do
@@ -166,7 +195,7 @@ class SampleJob
     ]
   end
 
-  define_transform :map_updates, sources: :all_contacts, targets: :contact_updates do
+  transform :map_updates do
     contact_updates.df = all_contacts.df[
       :Id,
       :Major__c

data/jobs/sftp_file_target_job.rb

@@ -1,18 +1,19 @@
 require_relative 'all_jobs_shared'
 
-class SftpFileTargetJob
-  include AllJobsShared
-
-
-  define_target :some_file, Remi::DataTarget::SftpFile,
-    credentials: {
-      host: 'example.com',
-      username: 'user',
-      password: 'secret'
-    },
-    local_path: "#{Remi::Settings.work_dir}/some_file.csv",
-    remote_path: "some_file_#{DateTime.current.strftime('%Y%m%d')}.csv"
+class SftpFileTargetJob < Remi::Job
+  target :some_file do
+    encoder Remi::Encoder::CsvFile.new
+    loader Remi::Loader::SftpFile.new(
+      credentials: {
+        host: 'example.com',
+        username: 'user',
+        password: 'secret'
+      },
+      local_path: "#{Remi::Settings.work_dir}/some_file.csv",
+      remote_path: "some_file_#{DateTime.current.strftime('%Y%m%d')}.csv"
+    )
+  end
 
-  define_transform :main do
+  transform :main do
   end
 end

data/jobs/sub_job_example_job.rb

@@ -0,0 +1,86 @@
+require_relative 'all_jobs_shared'
+
+class BeersJob < Remi::Job
+  source :beers do
+    extractor Remi::Extractor::DataFrame.new(
+      data: [
+        [ 'Baerlic', 'IPA' ],
+        [ 'Ex Novo', 'Red' ]
+      ]
+    )
+    parser Remi::Parser::DataFrame.new
+    fields(
+      {
+        brewer: {},
+        style: {}
+      }
+    )
+  end
+
+  transform :main do
+    # In the real world, add lots of complex stuff here, possibly grabbing
+    # from multiple sources.
+    beers.df
+  end
+end
+
+class ZombifyJob < Remi::Job
+  source :beers
+  target :zombie_beers
+
+  transform :main do
+    Remi::SourceToTargetMap.apply(beers.df, zombie_beers.df) do
+      map source(:brewer) .target(:brewer)
+        .transform(Remi::Transform::Prefix.new('Zombie '))
+      map source(:style) .target(:style)
+        .transform(Remi::Transform::Prefix.new('Zombie '))
+    end
+  end
+end
+
+class SubJobExampleJob < Remi::Job
+  sub_job(:beers_job) { BeersJob.new }
+  sub_job(:zombify_job) { ZombifyJob.new }
+
+  # This originates from a source in the sub job
+  source :beer_fridge do
+    extractor Remi::Extractor::SubJob.new(
+      sub_job: beers_job,
+      data_subject: :beers
+    )
+    fields beers_job.fields :beers
+  end
+
+  # This target is used as a source in the sub job
+  target :beers_to_zombify do
+    loader Remi::Loader::SubJob.new(
+      sub_job: zombify_job,
+      data_subject: :beers
+    )
+  end
+
+  # This source is obtained from the target of the sub job
+  source :zombie_fridge do
+    extractor Remi::Extractor::SubJob.new(
+      sub_job: zombify_job,
+      data_subject: :zombie_beers
+    )
+    fields zombify_job.fields :zombie_beers
+  end
+
+  # These are the ultimate targets of this job
+  target :just_beers
+  target :zombified_beers
+
+  transform :zombification do
+    # Sub jobs must be executed before their sources are available
+    beers_job.execute
+    just_beers.df = beer_fridge.df
+
+    # Sub job targets must be loaded before they are available to subjobs
+    beers_to_zombify.df = just_beers.df
+    beers_to_zombify.load
+    zombify_job.execute
+    zombified_beers.df = zombie_fridge.df
+  end
+end