ncs_mdes_warehouse 0.14.0 → 0.15.0.pre1

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.15.0
+------
+
+- Add soft_validations option: Record DataMapper validations
+  (as transform errors) but still attempt to export (#4315)
+
+- Add drop_not_null option: Drop NOT NULL constraints for all, but
+  the primary key colums in the database (#4316)
+
 0.14.0
 ------
 

data/lib/ncs_navigator/warehouse/cli.rb

@@ -4,10 +4,18 @@ require 'thor'
 
 module NcsNavigator::Warehouse
   class CLI < Thor
+
+    def initialize(args=[], options={}, config={})
+      super
+      @options = @options.dup
+    end
+
     class_option :quiet, :type => :boolean, :aliases => %w(-q),
       :desc => 'Suppress the status messages printed to standard error'
     class_option 'config', :type => :string, :aliases => %w(-c),
       :desc => "Supply an alternate configuration file instead of the default #{Configuration.environment_file}"
+    class_option :soft_validations, :type => :boolean,
+      :desc => "Record DataMapper validations (as transform errors) but still attempt to export"
 
     no_tasks {
       def configuration
@@ -21,6 +29,7 @@ module NcsNavigator::Warehouse
                 c.output_level = options['quiet'] ? :quiet : :normal
               end
               c.set_up_logs
+              c.soft_validations = options['soft_validations']
             end
           end
       end
@@ -89,10 +98,13 @@ DESC
       :desc => 'Copy the working schema to production even if there are errors'
     method_option 'preserve', :type => 'boolean',
       :desc => 'Do not wipe the working database before beginning ETL (for debugging)'
+    method_option 'drop_not_null', :type => 'boolean',
+      :desc => 'Drop NOT NULL constraints for all but the primary key columns in the database'
     def etl
       db = DatabaseInitializer.new(configuration)
       db.set_up_repository(:both)
       db.replace_schema unless options[:preserve]
+      db.drop_all_null_constraints(:working) if options[:drop_not_null]
 
       success = TransformLoad.new(configuration).run
       if success || options['force']
@@ -174,4 +186,3 @@ DESC
     end
   end
 end
-

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -406,6 +406,9 @@ module NcsNavigator::Warehouse
       @output_level ||= :normal
     end
 
+    # try to save records with DataMapper validation errors i.e. record.valid?
+    attr_accessor :soft_validations
+
     ##
     # Set the desired terminal output level.
     #
@@ -560,4 +563,3 @@ module NcsNavigator::Warehouse
     end
   end
 end
-

data/lib/ncs_navigator/warehouse/configuration/file_evaluator.rb

@@ -17,7 +17,7 @@ class NcsNavigator::Warehouse::Configuration
     end
 
     def eval
-      instance_eval(File.read(@filename), @filename)
+      instance_eval(File.read(@filename.to_s), @filename.to_s)
       configuration.finish
       @evaled = true
     end

data/lib/ncs_navigator/warehouse/database_initializer.rb

@@ -93,6 +93,26 @@ module NcsNavigator::Warehouse
       create_no_pii_views
     end
 
+    def drop_all_null_constraints(which, options={})
+      shell.clear_line_then_say "Dropping all NOT NULL constraints in #{which} schema" unless options[:quiet]
+      models = ::DataMapper::Model.descendants
+      adapter = ::DataMapper.repository(:"mdes_warehouse_#{which}").adapter
+
+      models.each do |model|
+        model.properties.each do |property|
+          # Skip if NULLS are allowed or column is part of the primary key
+          next if property.allow_blank? || model.key.include?(property)
+          adapter.execute(%Q|
+            ALTER TABLE #{model.storage_name}
+            ALTER COLUMN #{property.name}
+            DROP NOT NULL
+          |)
+        end
+      end
+
+      shell.clear_line_then_say "Dropped all NOT NULL constraints in #{which} schema.\n" unless options[:quiet]
+    end
+
     # @private Exposed for use in tests
     def drop_all(which, options={})
       shell.clear_line_then_say "Dropping everything in #{which} schema" unless options[:quiet]
@@ -252,4 +272,3 @@ module NcsNavigator::Warehouse
     private :escape_cmd
   end
 end
-

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -66,7 +66,7 @@ module NcsNavigator::Warehouse::Transformers
       @duplicates_strategy = select_duplicates_strategy
 
       @record_checkers = {
-        :validation => ValidateRecordChecker.new(log),
+        :validation => ValidateRecordChecker.new(log, @configuration),
         :foreign_key => ForeignKeyChecker.new(log, foreign_key_index),
         :psus => PsuIdChecker.new(log, @configuration.navigator.psus)
       }
@@ -139,11 +139,11 @@ module NcsNavigator::Warehouse::Transformers
       if saveable
         log.debug("Saving verified record #{record_ident record}.")
         begin
-          if record.save
+          if (@configuration.soft_validations ? record.save! : record.save)
             record
             foreign_key_index.record(record)
           else
-            msg = "Could not save valid record #{record.inspect}."
+            msg = "Could not save record #{record.inspect}.\n\t Errors: #{record.errors.inspect}"
             log.error msg
             status.unsuccessful_record(record, msg)
           end
@@ -225,8 +225,9 @@ module NcsNavigator::Warehouse::Transformers
 
       attr_reader :log
 
-      def initialize(log)
+      def initialize(log, configuration)
         @log = log
+        @configuration = configuration
       end
 
       def verify_or_report_errors(record, status)
@@ -244,7 +245,7 @@ module NcsNavigator::Warehouse::Transformers
               )
             end
           end
-          false
+          @configuration.soft_validations
         end
       end
 

data/lib/ncs_navigator/warehouse/transformers/event_start_from_contact_transformer.rb

@@ -50,12 +50,24 @@ module NcsNavigator::Warehouse::Transformers
     private
 
     # TODO: share this stuff between here and EnumTransformer
-
     def save(record, status)
-      if record.valid?
+      if !(valid = record.valid?)
+        log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
+        record.errors.keys.each do |prop|
+          record.errors[prop].each do |e|
+            status.unsuccessful_record(
+              record, "Invalid after transformation: #{e}.",
+              :attribute_name => prop,
+              :attribute_value => record.send(prop).inspect
+              )
+          end
+        end
+      end
+
+      if valid || @configuration.soft_validations
         log.debug("Saving valid transformed event record #{record_ident record}.")
         begin
-          unless record.save
+          unless (@configuration.soft_validations ? record.save! : record.save)
             msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
             log.error msg
             status.unsuccessful_record(record, msg)
@@ -65,17 +77,6 @@ module NcsNavigator::Warehouse::Transformers
           log.error msg
           status.unsuccessful_record(record, msg)
         end
-      else
-        log.error "Event invalid after transformation. #{record_messages(record).join(' ')}"
-        record.errors.keys.each do |prop|
-          record.errors[prop].each do |e|
-            status.unsuccessful_record(
-              record, "Invalid after transformation: #{e}.",
-              :attribute_name => prop,
-              :attribute_value => record.send(prop).inspect
-              )
-          end
-        end
       end
     end
 
@@ -95,4 +96,3 @@ module NcsNavigator::Warehouse::Transformers
     end
   end
 end
-

data/lib/ncs_navigator/warehouse/transformers/subprocess_transformer.rb

@@ -68,6 +68,7 @@ module NcsNavigator::Warehouse::Transformers
     def initialize(config, exec_and_args, options={})
       @configuration = config
       @exec_and_args = exec_and_args
+      @exec_and_args << '--soft_validations' if @configuration.soft_validations
       @directory = options[:directory] || '.'
       @environment = options[:environment] || {}
     end

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.14.0'
+    VERSION = '0.15.0.pre1'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -48,5 +48,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'yard', '~> 0.7.2'
   s.add_development_dependency 'ci_reporter', '1.6.6'
-  s.add_development_dependency 'fakefs', '0.4.0' # FakeFS does not follow semver
+  s.add_development_dependency 'fakefs', '0.4.2' # FakeFS does not follow semver
 end

data/spec/ncs_navigator/warehouse/database_initializer_spec.rb

@@ -2,6 +2,33 @@ require 'spec_helper'
 
 module NcsNavigator::Warehouse
   describe DatabaseInitializer do
+    describe '#drop_all_null_constraints', :slow do
+      it 'drops NOT NULL constraints on all non-primary key columns in all tables' do
+        subject = DatabaseInitializer.new(spec_config)
+        subject.set_up_repository(:working)
+        subject.replace_schema
+        subject.drop_all_null_constraints(:working)
+
+        models  = ::DataMapper::Model.descendants
+        adapter = ::DataMapper.repository(:mdes_warehouse_working).adapter
+
+        models.each do |m|
+          nullables = adapter.select(%Q{
+            SELECT cs.column_name
+              FROM information_schema.columns AS cs
+              WHERE cs.is_nullable = 'YES'
+              AND cs.table_name = '#{m.storage_name}'
+          })
+
+          m.properties.each do |p|
+            unless m.key.include?(p)
+              nullables.should include(p.name.to_s)
+            end
+          end
+        end
+      end
+    end
+
     describe '#replace_schema', :slow, :use_mdes do
       subject { DatabaseInitializer.new(spec_config) }
       let(:mdes_models) { spec_config.models_module.mdes_order }

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.14.0
-  prerelease: 
+  version: 0.15.0.pre1
+  prerelease: 7
 platform: ruby
 authors:
 - Rhett Sutphin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-15 00:00:00.000000000 Z
+date: 2013-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
@@ -452,7 +452,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -460,7 +460,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.2
 description: 
 email:
 - r-sutphin@northwestern.edu
@@ -3669,16 +3669,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3845842505256535190
+      hash: -4565215284799470018
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
-      segments:
-      - 0
-      hash: -3845842505256535190
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25