ncs_mdes_warehouse 0.0.2 → 0.1.0

data/lib/ncs_navigator/warehouse/transformers/vdr_xml.rb

@@ -9,7 +9,7 @@ module NcsNavigator::Warehouse::Transformers
       # @return [#transform] a transformer that loads the MDES data in
       #   the specified VDR XML file.
       def from_file(config, filename) # <- TODO better solution
-        EnumTransformer.new(Reader.new(config, filename))
+        EnumTransformer.new(config, Reader.new(config, filename))
       end
 
       ##

data/lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb

@@ -46,8 +46,9 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
       @start = Time.now
 
       Nokogiri::XML::Reader(@io).each do |node|
-        # 1 is an open element, 15 is a close
-        next unless [1, 15].include?(node.node_type)
+        element_types = [:TYPE_ELEMENT, :TYPE_END_ELEMENT].
+          collect { |c| Nokogiri::XML::Reader.const_get(c) }
+        next unless element_types.include?(node.node_type)
         encounter_node(node, node.node_type == 1, &block)
       end
 
@@ -77,8 +78,14 @@ class NcsNavigator::Warehouse::Transformers::VdrXml
           # node is the start tag of a table variable
           var = node.local_name.to_sym
           val = node.inner_xml.strip
-          node.read # skip contents (read as 'var' above)
-          node.read # skip close tag
+
+          unless node.self_closing?
+            # Skip to closing tag
+            n = node.read
+            until n.node_type == Nokogiri::XML::Reader::TYPE_END_ELEMENT
+              n = node.read
+            end
+          end
 
           unless should_filter_out(@current_model_class, var, val)
             @current_parameter_values[var] = val

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.0.2'
+    VERSION = '0.1.0'
   end
 end

data/spec/bcdatabase/test_sqlite.yml

@@ -3,3 +3,7 @@ defaults:
   datamapper_adapter: sqlite
 people_pro:
   database: people_pro.sqlite3
+mdes_warehouse_reporting:
+  database: wh_reporting.sqlite3
+mdes_warehouse_working:
+  database: wh_working.sqlite3

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -328,6 +328,48 @@ module NcsNavigator::Warehouse
       end
     end
 
+    describe '#pg_bin_path' do
+      it 'defaults to nil' do
+        config.pg_bin_path.should be_nil
+      end
+
+      describe 'when set' do
+        before do
+          config.pg_bin_path = '/Library/PostgreSQL/9.0/bin'
+        end
+
+        it 'is retrievable' do
+          config.pg_bin_path.to_s.should == '/Library/PostgreSQL/9.0/bin'
+        end
+
+        it 'becomes a Pathname' do
+          config.pg_bin_path.should be_a Pathname
+        end
+      end
+    end
+
+    describe '#pg_bin' do
+      describe 'with no path' do
+        before do
+          config.pg_bin_path = nil
+        end
+
+        it 'prepends nothing' do
+          config.pg_bin('pg_dumpall').to_s.should == 'pg_dumpall'
+        end
+      end
+
+      describe 'with a path' do
+        before do
+          config.pg_bin_path = '/home/me/bin'
+        end
+
+        it 'prepends the path' do
+          config.pg_bin('pg_dumpall').to_s.should == '/home/me/bin/pg_dumpall'
+        end
+      end
+    end
+
     describe '.from_file' do
       let(:filename) { tmpdir + 'test.rb' }
       subject { Configuration.from_file(filename) }

data/spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb

@@ -0,0 +1,187 @@
+require File.expand_path('../../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::PostgreSQL
+  describe Pgpass do
+    let(:home) { tmpdir('home') }
+    let(:file) { home + '.pgpass' }
+
+    before do
+      @original_home, ENV['HOME'] = ENV['HOME'], home
+    end
+
+    after do
+      ENV['HOME'] = @original_home
+    end
+
+    describe '#file' do
+      it 'defaults to $HOME/.pgpass' do
+        Pgpass.new.file.should == file
+      end
+    end
+
+    describe '.line' do
+      subject { Pgpass.line entry }
+      let(:entry) {
+        {
+          'username' => 'warehouse',
+          'password' => 'volt',
+          'database' => 'ignored'
+        }
+      }
+
+      it 'includes defaults for host and port if not set' do
+        subject.should == %w(localhost 5432 * warehouse volt)
+      end
+
+      it 'uses an explicit host if available' do
+        entry['host'] = 'ncs_db'
+        subject.should == %w(ncs_db 5432 * warehouse volt)
+      end
+
+      it 'uses an explicit port if available' do
+        entry['port'] = 5439
+        subject.should == %w(localhost 5439 * warehouse volt)
+      end
+
+      it 'fails without a username' do
+        entry.delete 'username'
+        lambda { subject }.should raise_error(/No username in configuration/)
+      end
+
+      it 'fails without a password' do
+        entry.delete 'password'
+        lambda { subject }.should raise_error(/No password in configuration/)
+      end
+    end
+
+    describe '#update' do
+      subject { Pgpass.new }
+      let(:entry) {
+        {
+          'username' => 'mw',
+          'password' => 'fishes',
+          'host' => 'ncsdb',
+          'port' => 5500,
+          'database' => 'mw_r'
+        }
+      }
+      let(:lines) {
+        file.readlines.collect { |l| l.chomp.split ':' }
+      }
+      let(:expected_line) { "ncsdb:5500:*:mw:fishes\n" }
+
+      describe 'when the file exists' do
+        describe 'and the same line exists' do
+          before do
+            file.open('w') do |f|
+              f.puts 'somewhere:5400:*:aguy:five'
+              f.puts 'ncsdb:5500:*:mw:fishes'
+              f.puts 'somewhere:5400:*:someoneelse:yet'
+            end
+
+            subject.update(entry)
+          end
+
+          it 'does nothing' do
+            lines.collect { |l| l[3] }.should == %w(aguy mw someoneelse)
+          end
+        end
+
+        describe 'and a similar line with a different password exists' do
+          before do
+            file.open('w') do |f|
+              f.puts 'somewhere:5400:*:aguy:five'
+              f.puts 'ncsdb:5500:*:mw:shark'
+              f.puts 'ncsdb:5500:*:someoneelse:fred'
+              f.puts 'somewhere:5400:*:someoneelse:yet'
+            end
+
+            subject.update(entry)
+          end
+
+          it 'updates the password' do
+            file.readlines[1].should == expected_line
+          end
+
+          it 'leaves other entries alone' do
+            lines.collect { |l| l[4] }.should == %w(five fishes fred yet)
+          end
+        end
+
+        describe 'and no similar entry exists' do
+          before do
+            file.open('w') do |f|
+              f.puts 'somewhere:5400:*:aguy:five'
+              f.puts 'ncsdb:5500:*:someoneelse:fred'
+              f.puts 'somewhere:5400:*:someoneelse:yet'
+            end
+
+            subject.update(entry)
+          end
+
+          it 'adds the entry' do
+            file.readlines.last.should == expected_line
+          end
+
+          it 'leaves the other entries alone' do
+            lines.collect { |l| l[0] }.should == %w(somewhere ncsdb somewhere ncsdb)
+          end
+        end
+      end
+
+      describe 'when the file does not exist' do
+        before do
+          file.exist?.should be_false
+
+          subject.update(entry)
+        end
+
+        it 'creates the file' do
+          file.exist?.should be_true
+        end
+
+        it 'sets the permissions to user-rw only' do
+          `ls -al '#{file}'`.should =~ /^-rw------/
+        end
+
+        it 'adds the appropriate entry' do
+          file.readlines.should == [expected_line]
+        end
+      end
+
+      describe 'when the directory does not exist' do
+        before do
+          home.rmdir
+        end
+
+        it 'fails' do
+          lambda { subject.update(entry) }.should(
+            raise_error %r{Cannot create #{home}/.pgpass})
+        end
+      end
+
+      describe 'when the directory is not writable' do
+        before do
+          home.chmod(0500)
+        end
+
+        it 'fails' do
+          lambda { subject.update(entry) }.should(
+            raise_error %r{Cannot create #{home}/.pgpass})
+        end
+      end
+
+      describe 'when the file exists but is not writable' do
+        before do
+          file.open('w') { |f| }
+          file.chmod(0500)
+        end
+
+        it 'fails' do
+          lambda { subject.update(entry) }.should(
+            raise_error %r{Cannot update #{home}/.pgpass})
+        end
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/table_modeler_spec.rb

@@ -449,8 +449,10 @@ end
         end
       end
 
+      let(:child_variable_name) { 'some_tree_id' }
+
       before do
-        table.variables << NcsNavigator::Mdes::Variable.new('some_tree_id').tap do |v|
+        table.variables << NcsNavigator::Mdes::Variable.new(child_variable_name).tap do |v|
           v.table_reference = parent_table
           v.type = NcsNavigator::Mdes::VariableType.new('foreignKeyRequiredType').tap do |vt|
             vt.base_type = :string
@@ -502,6 +504,18 @@ end
 
           instance.should_not be_valid
         end
+
+        describe 'when the child key does not have the "_id" suffix' do
+          let(:child_variable_name) { 'some_tree' }
+
+          it 'has an object reference accessor' do
+            model_class.instance_methods.collect(&:to_s).should include('some_tree_record')
+          end
+
+          it 'does not have a suffixed ID attribute' do
+            model_class.instance_methods.collect(&:to_s).should_not include('some_tree_id')
+          end
+        end
       end
     end
 

data/spec/ncs_navigator/warehouse/transform_load_spec.rb

@@ -0,0 +1,152 @@
+require File.expand_path('../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse
+  describe TransformLoad, :modifies_warehouse_state do
+    let(:config) { base_config }
+    let(:loader) { TransformLoad.new(config) }
+
+    def base_config
+      Configuration.new.tap do |c|
+        c.bcdatabase_group = :test_sqlite
+        c.log_file = tmpdir + "#{File.basename(__FILE__)}.log"
+        c.output_level = :quiet
+      end
+    end
+
+    before(:all) do
+      # Not config due to RSpec #500
+      DatabaseInitializer.new(base_config).tap do |dbi|
+        dbi.set_up_repository(:both)
+      end
+      TransformError.auto_migrate!(:mdes_warehouse_working)
+      TransformStatus.auto_migrate!(:mdes_warehouse_working)
+    end
+
+    describe '#run' do
+      it 'runs the transformers in the order specified' do
+        order = []
+        config.add_transformer(BlockTransformer.new { |s| order << 'B' })
+        config.add_transformer(BlockTransformer.new { |s| order << 'T' })
+        config.add_transformer(BlockTransformer.new { |s| order << 'w' })
+
+        loader.run
+
+        order.should == %w(B T w)
+      end
+
+      it 'passes a separate instance of TransformStatus to each transformer' do
+        statuses = []
+        config.add_transformer(BlockTransformer.new { |s| statuses << s })
+        config.add_transformer(BlockTransformer.new { |s| statuses << s })
+
+        loader.run
+
+        statuses.first.should_not be statuses.last
+      end
+
+      describe 'a created status' do
+        let(:statuses) { [] }
+        let(:status) { statuses.first }
+        let(:transformer) { BlockTransformer.new { |s| statuses << s } }
+
+        before do
+          config.add_transformer(transformer)
+        end
+
+        it 'uses the name provided by the transformer, if any' do
+          def transformer.name; 'provided'; end
+
+          loader.run
+
+          status.name.should == 'provided'
+        end
+
+        it 'uses the class name of the transformer otherwise' do
+          loader.run
+
+          status.name.should == 'BlockTransformer'
+        end
+
+        it 'sets the start and end times' do
+          Time.should_receive(:now).and_return(Time.iso8601('2001-01-12T08:00:00'))
+          Time.stub!(:now).and_return(Time.iso8601('2001-01-12T09:06:30'))
+
+          loader.run
+
+          status.start_time.hour.should == 8
+          status.end_time.hour.should == 9
+        end
+
+        it 'sets the position' do
+          loader.run
+          status.position.should == 0
+        end
+      end
+
+      it 'returns true if none of the transformers report an error' do
+        config.add_transformer(BlockTransformer.new { |s| })
+        loader.run.should be_true
+      end
+
+      describe 'with a reported error' do
+        let(:runs) { [] }
+
+        before do
+          config.add_transformer(BlockTransformer.new { |s| runs << 'A' })
+          config.add_transformer(BlockTransformer.new { |s|
+              runs << 'B'
+              s.unsuccessful_record(nil, 'fail')
+            })
+          config.add_transformer(BlockTransformer.new { |s| runs << 'C3' })
+          @result = loader.run
+        end
+
+        it 'still runs all the transformers' do
+          runs.should == %w(A B C3)
+        end
+
+        it 'returns false' do
+          @result.should == false
+        end
+      end
+
+      describe 'with a failure' do
+        let(:runs) { [] }
+
+        before do
+          config.add_transformer(BlockTransformer.new { |s| runs << 'A' })
+          config.add_transformer(BlockTransformer.new { |s|
+              runs << 'B'
+              fail 'No thanks.'
+            })
+          config.add_transformer(BlockTransformer.new { |s| runs << 'C3' })
+          @result = loader.run
+        end
+
+        it 'still runs all the transformers' do
+          runs.should == %w(A B C3)
+        end
+
+        it 'returns false' do
+          @result.should == false
+        end
+      end
+
+      describe 'sending e-mail' do
+        it 'sends on success'
+
+        it 'sends on failure'
+      end
+    end
+
+    class ::BlockTransformer
+      def initialize(&block)
+        @block = block
+      end
+
+      def transform(status)
+        @block.call(status)
+      end
+    end
+  end
+end