ncs_mdes_warehouse 0.2.0 → 0.3.0

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -1,8 +1,8 @@
 require 'ncs_navigator/warehouse'
 require 'ncs_navigator/configuration'
 require 'ncs_navigator/mdes'
+require 'ncs_navigator/warehouse/data_mapper'
 
-require 'data_mapper'
 require 'active_support/core_ext/object/try'
 require 'pathname'
 
@@ -30,7 +30,7 @@ module NcsNavigator::Warehouse
       #   given file.
       # @param [String] filename
       def from_file(filename)
-        FileEvaluator.new(filename).result
+        FileEvaluator.new(filename.to_s).result
       end
 
       ##
@@ -338,6 +338,27 @@ module NcsNavigator::Warehouse
       end
     end
 
+    ####
+    #### Meta
+    ####
+
+    ##
+    # The file from which this configuration was originally read. If
+    # the configuration was constructed solely in memory, this value
+    # will be `nil`.
+    #
+    # @return [Pathname,nil]
+    attr_reader :configuration_file
+
+    ##
+    # Specify the file from which this configuration was read.
+    #
+    # @param [Pathname,String,nil] fn
+    # @return [void]
+    def configuration_file=(fn)
+      @configuration_file = coerce_to_pathname(fn)
+    end
+
     private
 
     def coerce_to_pathname(value)

data/lib/ncs_navigator/warehouse/configuration/file_evaluator.rb

@@ -8,13 +8,17 @@ class NcsNavigator::Warehouse::Configuration
     alias :c :configuration
 
     def initialize(filename)
-      @filename = filename
+      @filename = Pathname.new(filename)
+      unless @filename.absolute?
+        @filename = @filename.realpath
+      end
       @base_configuration = NcsNavigator::Warehouse::Configuration.new
       @configuration = ErrorAccumulator.new(@base_configuration, @filename)
     end
 
     def eval
       instance_eval(File.read(@filename), @filename)
+      configuration.configuration_file = @filename
       configuration.finish
       @evaled = true
     end

data/lib/ncs_navigator/warehouse/data_mapper.rb

@@ -1,4 +1,9 @@
-require 'data_mapper'
+%w(
+  dm-core dm-constraints dm-migrations dm-transactions dm-validations dm-types dm-aggregates
+).each do |dm|
+  require dm
+end
+
 require 'ncs_navigator/warehouse'
 
 module NcsNavigator::Warehouse
@@ -14,6 +19,7 @@ module NcsNavigator::Warehouse
           accept_options :pii, :omittable
           attr_reader :pii, :omittable
         end
+        into.lazy(false)
       end
 
       def initialize(model, name, options = {})

data/lib/ncs_navigator/warehouse/data_mapper_patches.rb

@@ -4,6 +4,7 @@
 # been submitted to the dm-constraints:
 # https://github.com/datamapper/dm-constraints/pull/13
 
+require 'ncs_navigator/warehouse/data_mapper'
 require 'data_mapper/constraints/adapters/do_adapter'
 # @private
 module DataMapper::Constraints::Adapters::DataObjectsAdapter::SQL

data/lib/ncs_navigator/warehouse/database_initializer.rb

@@ -1,10 +1,10 @@
 require 'ncs_navigator/warehouse'
 
 require 'bcdatabase'
-require 'data_mapper'
 require 'benchmark'
 require 'forwardable'
 
+require 'ncs_navigator/warehouse/data_mapper'
 require 'ncs_navigator/warehouse/data_mapper_patches'
 
 module NcsNavigator::Warehouse

data/lib/ncs_navigator/warehouse/table_modeler/model_template.rb.erb

@@ -1,5 +1,5 @@
-require 'data_mapper'
 require 'ncs_navigator/warehouse'
+require 'ncs_navigator/warehouse/data_mapper'
 
 module <%= module_name %>
   class <%= t.wh_model_name %>

data/lib/ncs_navigator/warehouse/transform_status.rb

@@ -1,6 +1,5 @@
 require 'ncs_navigator/warehouse'
-
-require 'data_mapper'
+require 'ncs_navigator/warehouse/data_mapper'
 
 module NcsNavigator::Warehouse
   ##
@@ -27,7 +26,7 @@ module NcsNavigator::Warehouse
       storage_names[:mdes_warehouse_reporting] = 'wh_transform_status'
 
     property :id,           Serial
-    property :name,         String,  :required => true, :length => 255
+    property :name,         Text,    :required => true
     property :start_time,   DateTime
     property :end_time,     DateTime
     property :record_count, Integer, :default => 0

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -11,9 +11,10 @@ module NcsNavigator::Warehouse
   # `Enumerable` of MDES Warehouse model instances, validates and
   # saves them. It's a good general base for your own transformers.
   module Transformers
-    autoload :Database,        'ncs_navigator/warehouse/transformers/database'
-    autoload :EnumTransformer, 'ncs_navigator/warehouse/transformers/enum_transformer'
-    autoload :SamplingUnits,   'ncs_navigator/warehouse/transformers/sampling_units'
-    autoload :VdrXml,          'ncs_navigator/warehouse/transformers/vdr_xml'
+    autoload :Database,              'ncs_navigator/warehouse/transformers/database'
+    autoload :EnumTransformer,       'ncs_navigator/warehouse/transformers/enum_transformer'
+    autoload :SamplingUnits,         'ncs_navigator/warehouse/transformers/sampling_units'
+    autoload :SubprocessTransformer, 'ncs_navigator/warehouse/transformers/subprocess_transformer'
+    autoload :VdrXml,                'ncs_navigator/warehouse/transformers/vdr_xml'
   end
 end

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -1,8 +1,8 @@
 require 'ncs_navigator/warehouse'
+require 'ncs_navigator/warehouse/data_mapper'
 
 require 'active_support/core_ext/string'
 require 'forwardable'
-require 'data_mapper'
 
 module NcsNavigator::Warehouse::Transformers
   ##

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -42,7 +42,7 @@ module NcsNavigator::Warehouse::Transformers
           log.debug("Saving valid record #{record_ident record}.")
           begin
             unless record.save
-              msg = "Could not save."
+              msg = "Could not save valid record #{record.inspect}. #{record_messages(record).join(' ')}"
               log.error msg
               status.unsuccessful_record(record, msg)
             end
@@ -52,13 +52,7 @@ module NcsNavigator::Warehouse::Transformers
             status.unsuccessful_record(record, msg)
           end
         else
-          messages = record.errors.keys.collect { |prop|
-            record.errors[prop].collect { |e|
-              v = record.send(prop)
-              "#{e} (#{prop}=#{v.inspect})."
-            }
-          }.flatten
-          msg = "Invalid record. #{messages.join(' ')}"
+          msg = "Invalid record. #{record_messages(record).join(' ')}"
           log.error msg
           status.unsuccessful_record(record, msg)
         end
@@ -74,6 +68,15 @@ module NcsNavigator::Warehouse::Transformers
         rec.class.name.demodulize, rec.class.key.first.name, rec.key.try(:first).inspect]
     end
 
+    def record_messages(record)
+      record.errors.keys.collect { |prop|
+        record.errors[prop].collect { |e|
+          v = record.send(prop)
+          "#{e} (#{prop}=#{v.inspect})."
+        }
+      }.flatten
+    end
+
     def apply_global_values_if_necessary(record)
       {
         :psu_id => @configuration.navigator.psus.first.id,

data/lib/ncs_navigator/warehouse/transformers/subprocess_transformer.rb

@@ -0,0 +1,140 @@
+require 'ncs_navigator/warehouse'
+require 'childprocess'
+require 'forwardable'
+require 'json'
+require 'fileutils'
+
+module NcsNavigator::Warehouse::Transformers
+  ##
+  # A transformer that executes a separate executable to perform the
+  # transformation. The executable should provide error information
+  # on standard out. As it proceeds with the transformation, it can
+  # write single line messages. Each message either be an unstructured
+  # string or a single-line JSON object whose attributes are a subset
+  # of the attributes on {TransformError}. If the final line can be
+  # interpreted as an integer alone, it will be used as the record
+  # count for the transformer. Example standard out:
+  #
+  #     Could not connect to frob
+  #     { "record_id": "F111", "model_class": "Person", "message": "was invalid" }
+  #     39
+  #
+  # This output will result in a {TransformStatus} with record count
+  # 39 and two errors. The errors will have the messages `Could not
+  # connect to frob` and `was invalid` and the latter will have the
+  # identity of the specific troublesome record.
+  #
+  # If the subprocess is a ruby script, it may be useful to pass the
+  # warehouse configuration filename as one of its arguments so it can
+  # use the Warehouse API to connect to the database, etc.
+  class SubprocessTransformer
+    extend Forwardable
+
+    def_delegators :configuration, :log, :shell
+
+    ##
+    # @return [Array<String>] The process parameters, including the
+    #   executable name.
+    attr_reader :exec_and_args
+
+    ##
+    # @return [Configuration] The warehouse configuration in use.
+    attr_reader :configuration
+
+    ##
+    # @return [String] the working directory for the subprocess.
+    attr_reader :directory
+
+    ##
+    # @return [Hash<String, String>] supplemental environment variable
+    #   values for the subprocess.
+    attr_reader :environment
+
+    ##
+    # @param config [Configuration]
+    # @param exec_and_args [String<Array>] the arguments for the
+    #   subprocess to spawn, including the executable name (which
+    #   should be the first item in the array)
+    # @param options [Hash]
+    #
+    # @option options :environment [Hash<String, String>] values to
+    #   include in the subprocess's environment. The subprocess will
+    #   inherit all the environment variables from the parent ETL
+    #   process, so this only needs to include overrides.
+    # @option options :directory [String] the directory from which to
+    #   run the subprocess if different from the current working
+    #   directory.
+    def initialize(config, exec_and_args, options={})
+      @configuration = config
+      @exec_and_args = exec_and_args
+      @directory = options[:directory] || '.'
+      @environment = options[:environment] || {}
+    end
+
+    ##
+    # @return [String]
+    def name
+      exec_and_args.join(' ')
+    end
+
+    ##
+    # Spawns the child process and updates the provided
+    # {TransformStatus} with the results.
+    #
+    # @param [TransformStatus] transform_status
+    #
+    # @return [void]
+    def transform(transform_status)
+      process = ChildProcess.build(*exec_and_args)
+      process.environment.merge!(environment)
+
+      # inherit stderr
+      process.io.inherit!
+
+      # capture stdout
+      out_r, out_w = IO.pipe
+      process.io.stdout = out_w
+
+      log.info "Spawning subprocess transformer `#{exec_and_args.join(' ')}`"
+      shell.say "Spawning subprocess transformer `#{exec_and_args.join(' ')}`"
+
+      FileUtils.cd directory do
+        process.start
+      end
+
+      log.info "  PID is #{process.pid}"
+      shell.say_line ": #{process.pid}"
+      out_w.close
+
+      out_lines = out_r.readlines
+      if !out_lines.empty? && out_lines.last.to_i.to_s == out_lines.last.strip
+        transform_status.record_count = out_lines.pop.to_i
+      end
+      out_lines.each do |error|
+        attrs =
+          begin
+            JSON.parse(error.strip)
+          rescue JSON::ParserError => e
+            { 'message' => error.strip }
+          end
+        transform_status.transform_errors <<
+          NcsNavigator::Warehouse::TransformError.new(attrs)
+      end
+
+      unless process.exited?
+        process.stop
+      end
+
+      if process.exit_code != 0
+        transform_status.add_error(
+          "`#{exec_and_args.join(' ')}` process exited with code #{process.exit_code}")
+      end
+
+      if transform_status.transform_errors.empty?
+        shell.say_line "Subprocess completed."
+      else
+        shell.say_line "Subprocess completed with errors."
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.2.0'
+    VERSION = '0.3.0'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -23,17 +23,24 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'thor', '~> 0.14.6'
   s.add_dependency 'rubyzip', '~> 0.9.4'
+  s.add_dependency 'childprocess', '~> 0.2.3'
+  s.add_dependency 'json', '~> 1.6'
 
   s.add_dependency 'nokogiri', '~> 1.5.0'
   s.add_dependency 'builder', '>= 2.1.2'
 
-  s.add_dependency 'data_mapper', '~> 1.2.0'
+  %w(
+    dm-core dm-constraints dm-migrations dm-transactions dm-validations dm-types dm-aggregates
+    dm-postgres-adapter
+  ).each do |dm_gem|
+    s.add_dependency dm_gem, '~> 1.2.0'
+  end
+
   s.add_dependency 'bcdatabase', '~> 1.1'
-  s.add_dependency 'dm-postgres-adapter', '~> 1.2.0'
 
   s.add_development_dependency 'rspec', '~> 2.6'
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'yard', '~> 0.7.2'
-  s.add_development_dependency 'ci_reporter', '~> 1.6.5'
+  s.add_development_dependency 'ci_reporter', '1.6.6'
   s.add_development_dependency 'fakefs', '~> 0.4.0'
 end

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -394,6 +394,20 @@ module NcsNavigator::Warehouse
         subject.bcdatabase_group.should == :custom
       end
 
+      describe 'source file tracking' do
+        it 'knows what file it came from' do
+          write_file { }
+          subject.configuration_file.to_s.should == filename.to_s
+        end
+
+        it 'resolves relative paths' do
+          write_file { }
+          FileUtils.cd tmpdir do
+            Configuration.from_file('test.rb').configuration_file.to_s.should == filename.to_s
+          end
+        end
+      end
+
       describe 'with errors' do
         before do
           write_file do |f|

data/spec/ncs_navigator/warehouse/data_mapper_spec.rb

@@ -9,6 +9,10 @@ module NcsNavigator::Warehouse::DataMapper
     it 'uses the default method for other types' do
       subject.send(:typecast_to_primitive, 45).should == '45'
     end
+
+    it 'is not lazily-loaded' do
+      subject.options[:lazy].should be_false
+    end
   end
 
   describe NcsString, :modifies_warehouse_state do

data/spec/ncs_navigator/warehouse/models/mdes_model_spec.rb

@@ -1,7 +1,5 @@
 require File.expand_path('../../../../spec_helper', __FILE__)
 
-require 'data_mapper'
-
 module NcsNavigator::Warehouse::Models
   module Spec
     ##

data/spec/ncs_navigator/warehouse/table_modeler_spec.rb

@@ -1,7 +1,6 @@
 require File.expand_path('../../../spec_helper', __FILE__)
 
 require 'ncs_navigator/mdes'
-require 'data_mapper'
 
 module NcsNavigator::Warehouse
   module Spec

data/spec/ncs_navigator/warehouse/transform_status_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path('../../../spec_helper', __FILE__)
+
+module NcsNavigator::Warehouse
+  class SampleRecordishThing
+    include ::DataMapper::Resource
+    property :record_id, Serial, :key => true
+  end
+
+  SampleRecordishThing.finalize
+
+  describe TransformStatus do
+    subject { TransformStatus.memory_only('DC') }
+
+    describe '#unsuccessful_record' do
+      let(:rec) { SampleRecordishThing.new.tap { |t| t.record_id = 42 } }
+      let(:actual) { subject.transform_errors.last }
+
+      before do
+        subject.unsuccessful_record(rec, 'bad news')
+      end
+
+      it 'includes the record ID' do
+        actual.record_id.should == '42'
+      end
+
+      it 'includes the class' do
+        actual.model_class.should == SampleRecordishThing.to_s
+      end
+
+      it 'includes the message' do
+        actual.message.should == 'bad news'
+      end
+    end
+  end
+end