ncs_mdes_warehouse 0.4.1 → 0.5.0

data/lib/ncs_navigator/warehouse/transformers/subprocess_transformer.rb

@@ -3,6 +3,7 @@ require 'childprocess'
 require 'forwardable'
 require 'json'
 require 'fileutils'
+require 'bundler'
 
 module NcsNavigator::Warehouse::Transformers
   ##
@@ -99,7 +100,9 @@ module NcsNavigator::Warehouse::Transformers
       shell.say "Spawning subprocess transformer `#{exec_and_args.join(' ')}`"
 
       FileUtils.cd directory do
-        process.start
+        Bundler.with_clean_env do
+          process.start
+        end
       end
 
       log.info "  PID is #{process.pid}"

data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb

@@ -39,6 +39,14 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
       @record_count = 0
     end
 
+    def name
+      if @filename
+        "VDR XML #{@filename}"
+      else
+        "VDR XML #{@io.inspect}"
+      end
+    end
+
     ##
     # Reads the XML and yields each encountered record as a warehouse
     # model instance to the provided block. It does not validate the

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.4.1'
+    VERSION = '0.5.0'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
   end
 
   s.add_dependency 'bcdatabase', '~> 1.1'
+  s.add_dependency 'actionmailer', '~> 3.0'
 
   s.add_development_dependency 'rspec', '~> 2.6'
   s.add_development_dependency 'rake', '~> 0.9.2'

data/spec/navigator.ini

@@ -11,3 +11,6 @@ uri = "https://locahost/core"
 
 [PSC]
 uri = "https://localhost/psc"
+
+[SMTP]
+host = "smtp.ncs.gov"

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse
   describe Configuration do
@@ -41,6 +41,54 @@ module NcsNavigator::Warehouse
       end
     end
 
+    describe 'add_post_etl_hook' do
+      let(:success_hook) {
+        Object.new.tap do |o|
+          class << o
+            def etl_succeeded; end
+          end
+        end
+      }
+
+      let(:failure_hook) {
+        Object.new.tap do |o|
+          class << o
+            def etl_failed; end
+          end
+        end
+      }
+
+      it 'adds a hook with an `etl_succeeded` method' do
+        config.add_post_etl_hook(success_hook)
+        config.post_etl_hooks.should == [success_hook]
+      end
+
+      it 'adds a hook with an `etl_failed` method' do
+        config.add_post_etl_hook(failure_hook)
+        config.post_etl_hooks.should == [failure_hook]
+      end
+
+      describe 'with an object without either etl callback method' do
+        it 'gives a helpful message if the object is constructable' do
+          lambda { config.add_post_etl_hook(String) }.
+            should raise_error('String does not have an etl_succeeded or etl_failed method. Perhaps you meant String.new?')
+        end
+
+        it 'gives a helpful message if the object is an instance' do
+          lambda { config.add_post_etl_hook('not a good hook') }.
+            should raise_error('"not a good hook" does not have an etl_succeeded or etl_failed method.')
+        end
+
+        it 'does not add the object as a hook' do
+          begin
+            config.add_post_etl_hook('not a good hook')
+          rescue
+          end
+          config.post_etl_hooks.should be_empty
+        end
+      end
+    end
+
     describe '#mdes_version=' do
       context 'for a known version', :slow, :use_mdes, :modifies_warehouse_state  do
         it 'makes the models available' do
@@ -141,7 +189,10 @@ module NcsNavigator::Warehouse
 
     describe '#shell_io' do
       it 'is $stderr by default' do
-        config.shell_io.should be($stderr)
+        config.shell_io.object_id.should be($stderr.object_id)
+        # `should be($stderr)` and `should equal($stderr)` do not work
+        # with RSpec 2.10 and ci_reporter 1.6.6 because `should` is
+        # apparently delegated to the wrapped stream.
       end
 
       it 'can be set' do
@@ -281,6 +332,32 @@ module NcsNavigator::Warehouse
       end
     end
 
+    describe '#set_up_action_mailer' do
+      before do
+        config.set_up_action_mailer
+      end
+
+      it 'sets the SMTP settings from navigator.ini' do
+        ActionMailer::Base.smtp_settings[:address].should == 'smtp.ncs.gov'
+      end
+
+      it 'sets the delivery mode to SMTP' do
+        ActionMailer::Base.delivery_method.should == :smtp
+      end
+
+      it 'sets the template path appropriately' do
+        ActionMailer::Base.view_paths.collect(&:to_s).should == [
+          File.expand_path('../../../../lib/ncs_navigator/warehouse/mailer_templates', __FILE__)
+        ]
+      end
+
+      it 'only gets set up once' do
+        ActionMailer::Base.delivery_method = :test
+        config.set_up_action_mailer
+        ActionMailer::Base.delivery_method.should == :test
+      end
+    end
+
     describe 'bcdatabase' do
       describe 'group', :modifies_warehouse_state do
         subject { config.bcdatabase_group }

data/spec/ncs_navigator/warehouse/data_mapper_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse::DataMapper
   shared_examples 'a string-based NCS type' do

data/spec/ncs_navigator/warehouse/database_initializer_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse
   describe DatabaseInitializer do

data/spec/ncs_navigator/warehouse/hooks/etl_status_email_spec.rb

@@ -0,0 +1,126 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Hooks
+  describe EtlStatusEmail do
+    let(:config) {
+      NcsNavigator::Warehouse::Configuration.new.tap do |c|
+        c.log_file = tmpdir + "#{File.basename(__FILE__)}.log"
+        c.output_level = :quiet
+      end
+    }
+
+    let(:mailer) {
+      EtlStatusEmail.new(:to => to)
+    }
+
+    let(:to) {
+      ['jane@gcsc.net', 'joe@northwestern.edu']
+    }
+
+    let(:statuses) {
+      [
+        NcsNavigator::Warehouse::TransformStatus.memory_only('SP',
+          :start_time => Time.parse('2011-04-03T00:06:23'),
+          :end_time   => Time.parse('2011-04-03T00:53:00'),
+          :record_count => 5,
+          :position => 0),
+        NcsNavigator::Warehouse::TransformStatus.memory_only('Co',
+          :start_time => Time.parse('2011-04-03T00:53:01'),
+          :end_time   => Time.parse('2011-04-03T02:32:32'),
+          :record_count => 11,
+          :position => 1)
+      ]
+    }
+
+    let(:hook_args) {
+      {
+        :configuration => config,
+        :transform_statuses => statuses
+      }
+    }
+
+    let(:message) { ActionMailer::Base.deliveries.first }
+
+    before do
+      config.set_up_action_mailer
+      ActionMailer::Base.delivery_method = :test
+    end
+
+    after do
+      ActionMailer::Base.deliveries.clear
+    end
+
+    shared_examples 'an ETL result call' do
+      it 'sends an e-mail message' do
+        ActionMailer::Base.deliveries.size.should == 1
+      end
+
+      it 'sends the message to the configured addresses' do
+        message.to.should == to
+      end
+
+      it 'sends the message to the appropriate people based on their staff roles' do
+        pending 'would be nice: #2083'
+      end
+
+      describe 'the message content' do
+        it 'includes the duration' do
+          message.body.should =~ /2:26:09/
+        end
+
+        it 'includes the record count' do
+          message.body.should =~ /16 total records/
+        end
+
+        it 'includes a link to the warehouse dashboard' do
+          pending "there isn't one"
+        end
+      end
+    end
+
+    describe '#etl_succeeded' do
+      before do
+        mailer.etl_succeeded(hook_args)
+      end
+
+      include_examples 'an ETL result call'
+
+      describe 'the message subject' do
+        it 'indicates that the ETL succeeded' do
+          message.subject.should == '[NCS Navigator] Warehouse load successful'
+        end
+      end
+
+      describe 'the message content' do
+        it 'indicates that the ETL succeeded' do
+          message.body.should =~ /All transformations executed successfully./
+        end
+      end
+    end
+
+    describe '#etl_failed' do
+      before do
+        statuses[1].transform_errors <<
+          NcsNavigator::Warehouse::TransformError.new(:message => 'Not good enough')
+
+        mailer.etl_failed(hook_args)
+      end
+
+      include_examples 'an ETL result call'
+
+      describe 'the message subject' do
+        it 'indicates that the ETL failed' do
+          message.subject.should == '[NCS Navigator] Warehouse load failed'
+        end
+      end
+
+      describe 'the message content' do
+        it 'indicates that the ETL failed' do
+          message.body.
+            should =~ /1 transformation executed successfully.\s+1 transformation failed./
+        end
+      end
+    end
+  end
+end
+

data/spec/ncs_navigator/warehouse/models/mdes_model_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse::Models
   module Spec

data/spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse::PostgreSQL
   describe Pgpass do

data/spec/ncs_navigator/warehouse/stringify_trace_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse
+  describe StringifyTrace do
+    describe '#stringify_trace' do
+      it 'is available when mixed in' do
+        host = Class.new do
+          include StringifyTrace
+
+          def m
+            stringify_trace(['foo'])
+          end
+        end.new
+
+        lambda { host.m }.should_not raise_error
+      end
+    end
+
+    describe '.stringify_trace' do
+      let(:simple_trace) {
+        [
+          "(irb):2:in `irb_binding'",
+          "workspace.rb:80:in `eval'",
+          "foo/caller.rb:1024:in `<main>'"
+        ]
+      }
+
+      it 'aligns everything nicely' do
+        StringifyTrace.stringify_trace(simple_trace).should == <<-EXPECTED.chomp
+        (irb):   2:in `irb_binding'
+ workspace.rb:  80:in `eval'
+foo/caller.rb:1024:in `<main>'
+EXPECTED
+      end
+
+      it 'includes lines that cannot be parsed' do
+        StringifyTrace.stringify_trace(simple_trace + ['something up over here']).
+          should == <<-EXPECTED.chomp
+        (irb):   2:in `irb_binding'
+ workspace.rb:  80:in `eval'
+foo/caller.rb:1024:in `<main>'
+something up over here
+EXPECTED
+      end
+
+      it 'aligns lines without specific context info' do
+        StringifyTrace.stringify_trace(simple_trace + ['bar/zap.rb:423']).
+          should == <<-EXPECTED.chomp
+        (irb):   2:in `irb_binding'
+ workspace.rb:  80:in `eval'
+foo/caller.rb:1024:in `<main>'
+   bar/zap.rb: 423
+EXPECTED
+      end
+    end
+  end
+end
+

data/spec/ncs_navigator/warehouse/table_modeler_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 require 'ncs_navigator/mdes'
 

data/spec/ncs_navigator/warehouse/transform_load_spec.rb

@@ -1,4 +1,4 @@
-require File.expand_path('../../../spec_helper', __FILE__)
+require 'spec_helper'
 
 module NcsNavigator::Warehouse
   describe TransformLoad, :modifies_warehouse_state do
@@ -114,12 +114,13 @@ module NcsNavigator::Warehouse
 
       describe 'with a failure' do
         let(:runs) { [] }
+        let(:statuses) { TransformStatus.all(:order => [:id.desc]) }
 
         before do
           config.add_transformer(BlockTransformer.new { |s| runs << 'A' })
           config.add_transformer(BlockTransformer.new { |s|
               runs << 'B'
-              fail 'No thanks.'
+              fail 'No thanks'
             })
           config.add_transformer(BlockTransformer.new { |s| runs << 'C3' })
           @result = loader.run
@@ -132,6 +133,11 @@ module NcsNavigator::Warehouse
         it 'returns false' do
           @result.should == false
         end
+
+        it 'records the error including the trace' do
+          statuses[1].transform_errors.first.message.
+            should =~ /No thanks\n.*#{File.basename(__FILE__)}:\s*\d+/
+        end
       end
 
       describe 'with an integrity error on transaction commit' do
@@ -162,10 +168,86 @@ module NcsNavigator::Warehouse
         end
       end
 
-      describe 'sending e-mail' do
-        it 'sends on success'
+      describe 'with post-ETL hooks' do
+        let(:hook_a) { RecordingHook.new }
+        let(:hook_success) { RecordingHook.new(:succeeded) }
+        let(:hook_failure) { RecordingHook.new(:failed) }
+        let(:hook_error) {
+          Class.new do
+            def etl_succeeded(ts)
+              fail 'Hook broke'
+            end
+            alias :etl_failed :etl_succeeded
+          end.new
+        }
+
+        let(:hooks_invoked) {
+          [hook_a, hook_success, hook_failure].map(&:invoked?)
+        }
+
+        before do
+          config.add_post_etl_hook(hook_a)
+          config.add_post_etl_hook(hook_success)
+          config.add_post_etl_hook(hook_failure)
+        end
+
+        shared_examples 'all hooks' do
+          before do
+            loader.run
+          end
+
+          it 'sends each hook the statuses' do
+            hook_a.transform_statuses.size.should == 1
+          end
+
+          it 'sends each hook the configuration' do
+            hook_a.configuration.should == config
+          end
+
+          it 'runs all the hooks even when one fails' do
+            config.post_etl_hooks.unshift hook_error
+
+            loader.run
+
+            hooks_invoked.should == expected_invoke_pattern
+          end
+        end
+
+        describe 'when the transform fails' do
+          let(:expected_invoke_pattern) { [true, false, true] }
+
+          before do
+            config.add_transformer(BlockTransformer.new { |s| fail 'Nope.' })
+          end
+
+          include_examples 'all hooks'
 
-        it 'sends on failure'
+          it 'runs all the hooks that have an etl_failed method' do
+            hooks_invoked.should == expected_invoke_pattern
+          end
+
+          it 'sends each hook the overall status' do
+            hook_a.should be_failure
+          end
+        end
+
+        describe 'when the transform succeeds' do
+          let(:expected_invoke_pattern) { [true, true, false] }
+
+          before do
+            config.add_transformer(BlockTransformer.new { })
+          end
+
+          include_examples 'all hooks'
+
+          it 'runs all the hooks that have an etl_succeeded method' do
+            hooks_invoked.should == expected_invoke_pattern
+          end
+
+          it 'sends each hook the overall status' do
+            hook_a.should be_success
+          end
+        end
       end
 
       # This is a crappy test; it would be better if it could be done
@@ -195,5 +277,37 @@ module NcsNavigator::Warehouse
         @block.call(status)
       end
     end
+
+    class ::RecordingHook
+      attr_reader :transform_statuses, :configuration
+
+      def initialize(*modes)
+        @invoked = false
+        @modes = modes.empty? ? [:succeeded, :failed] : modes
+
+        @modes.each do |mode|
+          instance_eval <<-RUBY
+            def etl_#{mode}(args)
+              @invoked = true
+              @transform_statuses = args[:transform_statuses]
+              @configuration = args[:configuration]
+              @success = #{mode == :succeeded}
+            end
+          RUBY
+        end
+      end
+
+      def invoked?
+        @invoked
+      end
+
+      def success?
+        @success
+      end
+
+      def failure?
+        !@success.nil? && !@success
+      end
+    end
   end
 end