ncs_mdes_warehouse 0.0.2 → 0.1.0

data/CHANGELOG.md

@@ -1,6 +1,22 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.1.0
+-----
+
+- Implement actual ETL runner. It's accessible via the `etl`
+  subcommand of the `mdes-wh` executable.
+
+- Correct generated models so that association reference names and
+  foreign key column names do not collide. This was previously
+  possible when an MDES foreign key was not suffixed with "_id".
+
+- Replace `model_row` helper in `Transformers::Database` with its own
+  top-level production method, `produce_one_for_one`. The options and
+  behavior are mostly the same as `model_row`, but this refactoring
+  allows the results of the column mapping heuristic to be exposed to
+  assist in writing DRYer importers.
+
 0.0.2
 -----
 

data/generated_models/ncs_navigator/warehouse/models/two_point_zero/env_equipment_prob_log.rb

@@ -40,7 +40,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
     property   :equip_action,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :required => true, :pii => :possible, :length => 1..2, :set => ["1", "2", "3", "-7", "-4"] }
-    belongs_to :staff_id_reviewer,
+    belongs_to :staff_id_reviewer_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Staff',
                :child_key => [ :staff_id_reviewer ], :required => false
 

data/generated_models/ncs_navigator/warehouse/models/two_point_zero/incident.rb

@@ -34,25 +34,25 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
     belongs_to :inc_staff_supervisor,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Staff',
                :child_key => [ :inc_staff_supervisor_id ], :required => false
-    belongs_to :inc_recip_is_participant,
+    belongs_to :inc_recip_is_participant_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Participant',
                :child_key => [ :inc_recip_is_participant ], :required => false
-    belongs_to :inc_recip_is_du,
+    belongs_to :inc_recip_is_du_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::DwellingUnit',
                :child_key => [ :inc_recip_is_du ], :required => false
-    belongs_to :inc_recip_is_staff,
+    belongs_to :inc_recip_is_staff_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Staff',
                :child_key => [ :inc_recip_is_staff ], :required => false
-    belongs_to :inc_recip_is_family,
+    belongs_to :inc_recip_is_family_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Person',
                :child_key => [ :inc_recip_is_family ], :required => false
-    belongs_to :inc_recip_is_acquaintance,
+    belongs_to :inc_recip_is_acquaintance_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Person',
                :child_key => [ :inc_recip_is_acquaintance ], :required => false
     property   :inc_recip_is_other,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :format => /^([-+]?[\d]{1,9})?$/ }
-    belongs_to :inc_contact_person,
+    belongs_to :inc_contact_person_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Person',
                :child_key => [ :inc_contact_person ], :required => false
     property   :inctype,

data/generated_models/ncs_navigator/warehouse/models/two_point_zero/participant_rvis.rb

@@ -25,7 +25,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
     property   :rvis_language_oth,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :pii => :possible, :length => 0..255 }
-    belongs_to :rvis_person,
+    belongs_to :rvis_person_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Person',
                :child_key => [ :rvis_person ], :required => false
     property   :rvis_who_consented,

data/generated_models/ncs_navigator/warehouse/models/two_point_zero/sample_shipping.rb

@@ -40,7 +40,7 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
     property   :shipment_issues_oth,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :pii => :possible, :length => 0..255 }
-    belongs_to :staff_id_track,
+    belongs_to :staff_id_track_record,
                'NcsNavigator::Warehouse::Models::TwoPointZero::Staff',
                :child_key => [ :staff_id_track ], :required => false
     property   :sample_shipped_by,

data/lib/ncs_navigator/warehouse.rb

@@ -11,8 +11,12 @@ module NcsNavigator
     autoload :DataMapper,          'ncs_navigator/warehouse/data_mapper'
     autoload :DatabaseInitializer, 'ncs_navigator/warehouse/database_initializer'
     autoload :Models,              'ncs_navigator/warehouse/models'
+    autoload :PostgreSQL,          'ncs_navigator/warehouse/postgresql'
     autoload :TableModeler,        'ncs_navigator/warehouse/table_modeler'
     autoload :Transformers,        'ncs_navigator/warehouse/transformers'
+    autoload :TransformError,      'ncs_navigator/warehouse/transform_status'
+    autoload :TransformLoad,       'ncs_navigator/warehouse/transform_load'
+    autoload :TransformStatus,     'ncs_navigator/warehouse/transform_status'
     autoload :UpdatingShell,       'ncs_navigator/warehouse/updating_shell'
     autoload :VERSION,             'ncs_navigator/warehouse/version'
     autoload :XmlEmitter,          'ncs_navigator/warehouse/xml_emitter'

data/lib/ncs_navigator/warehouse/cli.rb

@@ -9,7 +9,7 @@ module NcsNavigator::Warehouse
     class_option :quiet, :type => :boolean, :aliases => %w(-q),
       :desc => 'Suppress the status messages printed to standard error'
     class_option 'config', :type => :string, :aliases => %w(-c),
-      :desc => 'Supply an alternate configuration file instead of the default /etc/nubic/ncs/warehouse/{env_name}.rb'
+      :desc => "Supply an alternate configuration file instead of the default #{Configuration.environment_file}"
 
     no_tasks {
       def configuration
@@ -40,6 +40,12 @@ module NcsNavigator::Warehouse
       db.replace_schema
     end
 
+    desc 'clone-working', 'Copies the contents of the working database to the reporting database'
+    def clone_working
+      db = DatabaseInitializer.new(configuration)
+      db.clone_working_to_reporting
+    end
+
     desc 'emit-xml [FILENAME]', 'Generates the VDR submission XML'
     long_desc <<-DESC
 Generates and zips the vanguard data repository submission XML from
@@ -52,6 +58,30 @@ DESC
 
       XmlEmitter.new(configuration, filename).emit_xml
     end
+
+    desc 'etl', 'Performs the full extract-transform-load process for this configuration'
+    long_desc <<-DESC
+Clears the working schema and repopulates it with the results of running
+the all the configured transforms. If the transforms are successful, the
+reporting schema is wiped and replaced with the results.
+DESC
+    method_option 'force', :type => 'boolean',
+      :desc => 'Copy the working schema to production even if there are errors'
+    def etl
+      db = DatabaseInitializer.new(configuration)
+      db.set_up_repository(:both)
+      db.replace_schema
+
+      success = TransformLoad.new(configuration).run
+      if success || options['force']
+        db.clone_working_to_reporting
+      else
+        configuration.shell.say_line "There were errors during ETL. Reporting database not updated."
+        configuration.shell.say_line "See the log and the database table wh_transform_error for more details."
+
+        exit 1
+      end
+    end
   end
 end
 

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -2,6 +2,7 @@ require 'ncs_navigator/warehouse'
 require 'ncs_navigator/configuration'
 require 'ncs_navigator/mdes'
 
+require 'data_mapper'
 require 'active_support/core_ext/object/try'
 require 'pathname'
 
@@ -276,15 +277,7 @@ module NcsNavigator::Warehouse
     #
     # @param [Pathname,String,nil] fn
     def log_file=(fn)
-      @log_directory =
-        case fn
-        when nil
-          nil
-        when Pathname
-          fn
-        else
-          Pathname.new(fn)
-        end
+      @log_directory = coerce_to_pathname(fn)
     end
 
     def set_up_logs
@@ -310,6 +303,53 @@ module NcsNavigator::Warehouse
       set_up_logs unless @log
       @log
     end
+
+    ####
+    #### pg_bin
+    ####
+
+    ##
+    # The path where the PostgreSQL command line utilities can be
+    # found. If they are on the search path, this may be `nil` (the
+    # default).
+    #
+    # @return [Pathname, nil]
+    attr_reader :pg_bin_path
+
+    ##
+    # Specify the path where the PostgreSQL command line utilities can
+    # be found.
+    #
+    # @param [Pathname,String,nil] fn
+    # @return [void]
+    def pg_bin_path=(fn)
+      @pg_bin_path = coerce_to_pathname(fn)
+    end
+
+    ##
+    # @return [Pathname] the executable for the given PostgreSQL
+    #   utility.
+    # @param [Pathname, String] command the name of the command
+    def pg_bin(command)
+      if pg_bin_path
+        pg_bin_path + command
+      else
+        coerce_to_pathname command
+      end
+    end
+
+    private
+
+    def coerce_to_pathname(value)
+      case value
+      when nil
+        nil
+      when Pathname
+        value
+      else
+        Pathname.new(value)
+      end
+    end
   end
 end
 

data/lib/ncs_navigator/warehouse/database_initializer.rb

@@ -57,6 +57,8 @@ module NcsNavigator::Warehouse
     def connect_one(which_one, dm_name=nil)
       dm_name ||= :"mdes_warehouse_#{which_one}"
       log.info "Connecting DataMapper repository #{dm_name.inspect}"
+      p = params(which_one)
+      log.debug "  using #{p.merge('password' => 'SUPPRESSED').inspect}"
       adapter = ::DataMapper.setup(dm_name, params(which_one))
     end
     private :connect_one
@@ -78,7 +80,6 @@ module NcsNavigator::Warehouse
     #
     # @return [void]
     def replace_schema
-      # TODO: actual logging, too
       shell.say "Dropping everything"
       log.info "Dropping everything in working schema"
       ::DataMapper.repository(:mdes_warehouse_working).adapter.
@@ -89,12 +90,69 @@ module NcsNavigator::Warehouse
       log.info "Initializing schema for MDES #{configuration.mdes.specification_version}"
       # In DM 1.2, DataMapper.auto_migrate! only works for the
       # :default repo
-      configuration.models_module.mdes_order.each do |m|
-        shell.clear_line_then_say "Adding #{m.mdes_table_name}..."
+      ::DataMapper::Model.descendants.each do |m|
+        shell.clear_line_then_say "Adding #{m.storage_name(:mdes_warehouse_working)}..."
         m.auto_migrate!(:mdes_warehouse_working)
       end
-      shell.clear_line_then_say "Added #{configuration.models_module.mdes_order.size} tables.\n"
+      shell.clear_line_then_say(
+        "Added #{configuration.models_module.mdes_order.size} MDES tables.\n")
     end
+
+    ##
+    # Replaces the reporting database with a clone of the working
+    # database. This method relies on the command line `pg_dump` and
+    # `pg_restore` commands.
+    #
+    # @see Configuration#pg_bin_path
+    def clone_working_to_reporting
+      PostgreSQL::Pgpass.new.tap do |pgpass|
+        pgpass.update params(:working)
+        pgpass.update params(:reporting)
+      end
+
+      dump_cmd = [
+        configuration.pg_bin('pg_dump'),
+        pg_params(params(:working)),
+        '--format=custom',
+        params(:working)['database']
+      ].flatten
+
+      restore_cmd = [
+        configuration.pg_bin('pg_restore'),
+        pg_params(params(:reporting)),
+        '--schema', 'public',
+        '--clean',
+        '--dbname', params(:reporting)['database']
+      ].flatten
+
+      command = "#{escape_cmd dump_cmd} | #{escape_cmd restore_cmd}"
+      log.info('Cloning working schema into reporting schema')
+      log.debug("Clone command: #{command.inspect}")
+      unless system(command)
+        configuration.shell.say_line "Clone from working to reporting failed. See above for detail."
+        exit 1
+      end
+    end
+
+    def pg_params(p)
+      [
+        pg_param(p, 'host'),
+        pg_param(p, 'port'),
+        pg_param(p, 'username'),
+        '-w'
+      ].compact.flatten
+    end
+    private :pg_params
+
+    def pg_param(p, param_name)
+      ["--#{param_name}", p[param_name]] if p[param_name]
+    end
+    private :pg_param
+
+    def escape_cmd(parts)
+      parts.collect { |p| "'#{p}'" }.join(' ')
+    end
+    private :escape_cmd
   end
 end
 

data/lib/ncs_navigator/warehouse/models.rb

@@ -1,5 +1,8 @@
 require 'ncs_navigator/warehouse'
 
+# ensure that this model is loaded along with the others
+require 'ncs_navigator/warehouse/transform_status'
+
 module NcsNavigator::Warehouse
   module Models
     autoload :MdesModel,           'ncs_navigator/warehouse/models/mdes_model'

data/lib/ncs_navigator/warehouse/postgresql.rb

@@ -0,0 +1,7 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  module PostgreSQL
+    autoload :Pgpass, 'ncs_navigator/warehouse/postgresql/pgpass'
+  end
+end

data/lib/ncs_navigator/warehouse/postgresql/pgpass.rb

@@ -0,0 +1,79 @@
+require 'ncs_navigator/warehouse'
+
+require 'pathname'
+require 'forwardable'
+
+module NcsNavigator::Warehouse::PostgreSQL
+  class Pgpass
+    extend Forwardable
+
+    ##
+    # Converts a database configuration hash into the elements of the
+    # corresponding .pgpass line.
+    #
+    # @param [Hash<String, String>] the configuration
+    # @return [Array<String>]
+    def self.line(entry)
+      [
+        entry['host'] || 'localhost',
+        (entry['port'] || 5432).to_s,
+        '*',
+        entry['username'] || fail("No username in configuration #{entry.inspect}"),
+        entry['password'] || fail("No password in configuration #{entry.inspect}")
+      ]
+    end
+
+    def_delegator self, :line
+
+    attr_reader :file
+
+    def initialize
+      @file = Pathname.new(ENV['HOME']) + '.pgpass'
+    end
+
+    ##
+    # Updates the `.pgpass` file so that it includes the current
+    # password for the given configuration. This may involve adding an
+    # entry, replacing an entry, or even creating the file entirely.
+    #
+    # @param [Hash] entry a database configuration hash
+    # @return [void]
+    def update(entry)
+      ensure_file_exists_and_is_writable
+
+      new_line = self.line(entry)
+      contents = file.readlines.collect { |l| l.chomp.split(':') }
+      match = contents.detect { |line| line[0..3] == new_line[0..3] }
+
+      if match
+        match[4] = entry['password']
+      else
+        contents << new_line
+      end
+
+      file.open('w') do |f|
+        contents.each do |l|
+          f.puts l.join(':')
+        end
+      end
+      file.chmod(0600)
+    end
+
+    private
+
+    def ensure_file_exists_and_is_writable
+      if file.exist?
+        if file.writable?
+          # do nothing
+        else
+          fail "Cannot update #{file}"
+        end
+      elsif file.parent.writable?
+        # touch
+        file.open('w') { |f| }
+      else
+        fail "Cannot create #{file}"
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/table_modeler/mdes_ext.rb

@@ -73,6 +73,15 @@ module NcsNavigator
           end
         ].compact
       end
+
+      def wh_reference_name
+        fail 'Does not apply' unless self.table_reference
+        if self.name =~ /_id$/
+          self.name.sub(/_id$/, '')
+        else
+          self.name + '_record'
+        end
+      end
     end
 
     class VariableType

data/lib/ncs_navigator/warehouse/table_modeler/model_template.rb.erb

@@ -12,7 +12,7 @@ module <%= module_name %>
 
 <% t.wh_variables.each do |v| -%>
 <% if v.table_reference -%>
-    belongs_to :<%= v.name.sub(/_id$/, '') %>,
+    belongs_to :<%= v.wh_reference_name %>,
                '<%= v.table_reference.wh_model_name(module_name) %>',
                :child_key => [ :<%= v.name %> ], :required => <%= !!v.required %>
 <% else -%>