ncs_mdes_warehouse 0.2.0 → 0.3.0

data/spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb

@@ -1,5 +1,7 @@
 require File.expand_path('../../../../spec_helper', __FILE__)
 
+require 'ncs_navigator/warehouse/data_mapper'
+
 module NcsNavigator::Warehouse::Transformers
   describe EnumTransformer, :modifies_warehouse_state do
     class Sample
@@ -97,7 +99,7 @@ module NcsNavigator::Warehouse::Transformers
           err = transform_status.transform_errors.first
           err.model_class.should == Sample.to_s
           err.record_id.should == '2'
-          err.message.should == 'Could not save.'
+          err.message.should =~ /^Could not save valid record/
         end
 
         it 'saves the saveable instances' do

data/spec/ncs_navigator/warehouse/transformers/subprocess_transformer_spec.rb

@@ -0,0 +1,227 @@
+require File.expand_path('../../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse::Transformers
+  describe SubprocessTransformer do
+    let(:script_name) { tmpdir + 'subprocess_test_script.sh' }
+    let(:transform_status) { NcsNavigator::Warehouse::TransformStatus.memory_only(script_name) }
+    let(:config) {
+      NcsNavigator::Warehouse::Configuration.new.tap do |c|
+        c.output_level = :quiet
+        c.log_file = tmpdir + 'spec.log'
+      end
+    }
+    let(:options) { { } }
+    let(:messages) { transform_status.transform_errors.collect(&:message) }
+
+    subject { SubprocessTransformer.new(config, ['bash', script_name.to_s], options) }
+
+    def write_script
+      File.open(script_name, 'w') do |f|
+        yield f
+      end
+    end
+
+    describe '#name' do
+      it 'is the command line' do
+        subject.name.should == "bash #{script_name}"
+      end
+    end
+
+    describe '#transform' do
+      let(:side_file) { tmpdir + 'quux.foo' }
+
+      it 'executes the script' do
+        write_script do |f|
+          f.puts %(echo "bar" > '#{side_file}')
+        end
+
+        subject.transform(transform_status)
+
+        side_file.read.should == "bar\n"
+      end
+    end
+
+    describe 'status updating' do
+      describe 'with JSON errors' do
+        before do
+          write_script do |f|
+            f.puts %(echo '{ "message": "A bad bad thing" }')
+            f.puts %(echo '{ "message": "A record-specific bad thing", "record_id": "foo11", "model_class": "Person" }')
+            f.puts %(echo 93)
+          end
+
+          subject.transform(transform_status)
+        end
+
+        it 'appends the errors to the status' do
+          transform_status.should have(2).transform_errors
+        end
+
+        it 'updates the record count from the last line' do
+          transform_status.record_count.should == 93
+        end
+
+        describe 'error detail' do
+          let(:error0) { transform_status.transform_errors[0] }
+          let(:error1) { transform_status.transform_errors[1] }
+
+          it 'includes the message' do
+            error0.message.should == 'A bad bad thing'
+          end
+
+          it 'includes the record_id' do
+            error1.record_id.should == 'foo11'
+          end
+
+          it 'includes the model class' do
+            error1.model_class.should == 'Person'
+          end
+        end
+      end
+
+      describe 'with no errors' do
+        before do
+          write_script do |f|
+            f.puts %(echo 77)
+          end
+
+          subject.transform(transform_status)
+        end
+
+        it 'updates the record count' do
+          transform_status.record_count.should == 77
+        end
+
+        it 'has no errors' do
+          transform_status.should have(0).transform_errors
+        end
+      end
+
+      describe 'with no output' do
+        before do
+          write_script { }
+          subject.transform(transform_status)
+        end
+
+        it 'has no records' do
+          transform_status.record_count.should == 0
+        end
+
+        it 'has no errors' do
+          transform_status.should have(0).transform_errors
+        end
+      end
+
+      describe 'with non-JSON errors' do
+        before do
+          write_script do |f|
+            f.puts %(echo 'A bad bad thing')
+            f.puts %(echo 'Another bad thing')
+            f.puts %(echo 90)
+          end
+
+          subject.transform(transform_status)
+        end
+
+        it 'has the errors' do
+          messages.should == [
+            "A bad bad thing", "Another bad thing"
+          ]
+        end
+
+        it 'has the record count' do
+          transform_status.record_count.should == 90
+        end
+      end
+
+      describe 'with a nonzero exit' do
+        before do
+          write_script do |f|
+            f.puts %(echo '{ "message": "A bad bad thing" }')
+            f.puts %(echo 'Another bad thing')
+            f.puts %(exit 4)
+          end
+
+          subject.transform(transform_status)
+        end
+
+        it 'has any errors that were printed before the exit' do
+          messages[0..1].should == [
+            "A bad bad thing", "Another bad thing"
+          ]
+        end
+
+        it 'has no record count' do
+          transform_status.record_count.should == 0
+        end
+
+        it 'has an error about the exit' do
+          messages[2].should == "`bash #{script_name}` process exited with code 4"
+        end
+      end
+    end
+
+    describe 'working directory' do
+      let(:dir) { tmpdir('bar') }
+
+      before do
+        write_script do |f|
+          f.puts 'echo `pwd`'
+        end
+
+        options[:directory] = dir
+
+        subject.transform(transform_status)
+      end
+
+      it 'switches to the specified directory before running' do
+        messages.first.should == dir.to_s
+      end
+    end
+
+    describe 'environment' do
+      before do
+        write_script do |f|
+          f.puts 'echo $SOME_ENV'
+        end
+
+        options[:environment] = { 'SOME_ENV' => 'staging' }
+
+        subject.transform(transform_status)
+      end
+
+      it 'has the specified values' do
+        messages.first.should == 'staging'
+      end
+
+      it 'does not monkey with the parent environment' do
+        ENV['SOME_ENV'].should be_nil
+      end
+    end
+
+    describe 'standard error' do
+      let(:replacement_stderr) { StringIO.new }
+
+      before do
+        pending 'not sure this is desirable'
+
+        write_script do |f|
+          f.puts 'echo "On SE" 1>&2'
+        end
+
+        begin
+          old_err, $stderr = $stderr, replacement_stderr
+          subject.transform(transform_status)
+        ensure
+          $stderr = old_err
+        end
+      end
+
+      it 'does not go to the parent stream' do
+        replacement_stderr.string.should == ''
+      end
+
+      it 'goes to the log'
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -10,7 +10,7 @@ $LOAD_PATH.push File.expand_path('..', __FILE__)
 
 require 'global_state_helper'
 
-require 'data_mapper'
+require 'dm-core'
 
 RSpec.configure do |config|
   config.treat_symbols_as_metadata_keys_with_true_values = true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-11 00:00:00.000000000Z
+date: 2011-12-18 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
-  requirement: &2152563420 !ruby/object:Gem::Requirement
+  requirement: &2152835540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *2152563420
+  version_requirements: *2152835540
 - !ruby/object:Gem::Dependency
   name: ncs_navigator_configuration
-  requirement: &2152561100 !ruby/object:Gem::Requirement
+  requirement: &2152834920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.2'
   type: :runtime
   prerelease: false
-  version_requirements: *2152561100
+  version_requirements: *2152834920
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &2152559400 !ruby/object:Gem::Requirement
+  requirement: &2152834360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '3.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152559400
+  version_requirements: *2152834360
 - !ruby/object:Gem::Dependency
   name: i18n
-  requirement: &2152558860 !ruby/object:Gem::Requirement
+  requirement: &2152833540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '0.4'
   type: :runtime
   prerelease: false
-  version_requirements: *2152558860
+  version_requirements: *2152833540
 - !ruby/object:Gem::Dependency
   name: thor
-  requirement: &2152558360 !ruby/object:Gem::Requirement
+  requirement: &2152832960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.14.6
   type: :runtime
   prerelease: false
-  version_requirements: *2152558360
+  version_requirements: *2152832960
 - !ruby/object:Gem::Dependency
   name: rubyzip
-  requirement: &2152557580 !ruby/object:Gem::Requirement
+  requirement: &2152832420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,32 @@ dependencies:
         version: 0.9.4
   type: :runtime
   prerelease: false
-  version_requirements: *2152557580
+  version_requirements: *2152832420
+- !ruby/object:Gem::Dependency
+  name: childprocess
+  requirement: &2152831800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152831800
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2152831020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152831020
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2152556800 !ruby/object:Gem::Requirement
+  requirement: &2152830380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +109,10 @@ dependencies:
         version: 1.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152556800
+  version_requirements: *2152830380
 - !ruby/object:Gem::Dependency
   name: builder
-  requirement: &2152556200 !ruby/object:Gem::Requirement
+  requirement: &2152829800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +120,10 @@ dependencies:
         version: 2.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *2152556200
+  version_requirements: *2152829800
 - !ruby/object:Gem::Dependency
-  name: data_mapper
-  requirement: &2152555680 !ruby/object:Gem::Requirement
+  name: dm-core
+  requirement: &2152828780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,21 +131,76 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152555680
+  version_requirements: *2152828780
 - !ruby/object:Gem::Dependency
-  name: bcdatabase
-  requirement: &2152555140 !ruby/object:Gem::Requirement
+  name: dm-constraints
+  requirement: &2152828020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152555140
+  version_requirements: *2152828020
+- !ruby/object:Gem::Dependency
+  name: dm-migrations
+  requirement: &2152827020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152827020
+- !ruby/object:Gem::Dependency
+  name: dm-transactions
+  requirement: &2152825700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152825700
+- !ruby/object:Gem::Dependency
+  name: dm-validations
+  requirement: &2152816120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152816120
+- !ruby/object:Gem::Dependency
+  name: dm-types
+  requirement: &2152814800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152814800
+- !ruby/object:Gem::Dependency
+  name: dm-aggregates
+  requirement: &2152813800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152813800
 - !ruby/object:Gem::Dependency
   name: dm-postgres-adapter
-  requirement: &2152554620 !ruby/object:Gem::Requirement
+  requirement: &2152812620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,10 +208,21 @@ dependencies:
         version: 1.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2152554620
+  version_requirements: *2152812620
+- !ruby/object:Gem::Dependency
+  name: bcdatabase
+  requirement: &2152758260 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2152758260
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152553900 !ruby/object:Gem::Requirement
+  requirement: &2152757000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -142,10 +230,10 @@ dependencies:
         version: '2.6'
   type: :development
   prerelease: false
-  version_requirements: *2152553900
+  version_requirements: *2152757000
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2152553140 !ruby/object:Gem::Requirement
+  requirement: &2152755580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -153,10 +241,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *2152553140
+  version_requirements: *2152755580
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &2152552420 !ruby/object:Gem::Requirement
+  requirement: &2152754420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -164,21 +252,21 @@ dependencies:
         version: 0.7.2
   type: :development
   prerelease: false
-  version_requirements: *2152552420
+  version_requirements: *2152754420
 - !ruby/object:Gem::Dependency
   name: ci_reporter
-  requirement: &2152551400 !ruby/object:Gem::Requirement
+  requirement: &2152753160 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - =
       - !ruby/object:Gem::Version
-        version: 1.6.5
+        version: 1.6.6
   type: :development
   prerelease: false
-  version_requirements: *2152551400
+  version_requirements: *2152753160
 - !ruby/object:Gem::Dependency
   name: fakefs
-  requirement: &2152550660 !ruby/object:Gem::Requirement
+  requirement: &2152752120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -186,7 +274,7 @@ dependencies:
         version: 0.4.0
   type: :development
   prerelease: false
-  version_requirements: *2152550660
+  version_requirements: *2152752120
 description: 
 email:
 - r-sutphin@northwestern.edu
@@ -493,6 +581,7 @@ files:
 - lib/ncs_navigator/warehouse/transformers/database.rb
 - lib/ncs_navigator/warehouse/transformers/enum_transformer.rb
 - lib/ncs_navigator/warehouse/transformers/sampling_units.rb
+- lib/ncs_navigator/warehouse/transformers/subprocess_transformer.rb
 - lib/ncs_navigator/warehouse/transformers/vdr_xml.rb
 - lib/ncs_navigator/warehouse/transformers/vdr_xml/reader.rb
 - lib/ncs_navigator/warehouse/updating_shell.rb
@@ -511,9 +600,11 @@ files:
 - spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb
 - spec/ncs_navigator/warehouse/table_modeler_spec.rb
 - spec/ncs_navigator/warehouse/transform_load_spec.rb
+- spec/ncs_navigator/warehouse/transform_status_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/sampling_units_spec.rb
+- spec/ncs_navigator/warehouse/transformers/subprocess_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml_spec.rb
@@ -534,12 +625,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1071388542021562358
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 1071388542021562358
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10
@@ -559,9 +656,11 @@ test_files:
 - spec/ncs_navigator/warehouse/postgresql/pgpass_spec.rb
 - spec/ncs_navigator/warehouse/table_modeler_spec.rb
 - spec/ncs_navigator/warehouse/transform_load_spec.rb
+- spec/ncs_navigator/warehouse/transform_status_spec.rb
 - spec/ncs_navigator/warehouse/transformers/database_spec.rb
 - spec/ncs_navigator/warehouse/transformers/enum_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/sampling_units_spec.rb
+- spec/ncs_navigator/warehouse/transformers/subprocess_transformer_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/made_up_vdr_xml.xml
 - spec/ncs_navigator/warehouse/transformers/vdr_xml/reader_spec.rb
 - spec/ncs_navigator/warehouse/transformers/vdr_xml_spec.rb