ncs_mdes_warehouse 0.9.0 → 0.10.0

data/CHANGELOG.md

@@ -1,6 +1,15 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.10.0
+------
+
+- Add MDES CSV transformer. (#2710)
+
+- Enable use in Rails 3.2.7+ applications. ActiveSupport 3.2.4-3.2.6 had [an
+  issue](https://github.com/rails/rails/pull/6857) which caused a conflict with
+  DataMapper. This issue was fixed in 3.2.7.
+
 0.9.0
 -----
 

data/Gemfile

@@ -6,7 +6,7 @@ gemspec
 # activesupport, and builder, bundler takes infinite time to resolve unless you
 # lock it down a little.
 group :resolver_hint do
-  gem 'actionpack', '~> 3.1.8'
+  gem 'actionpack', '~> 3.2.8'
 end
 
 group :development do

data/lib/ncs_navigator/warehouse/cli.rb

@@ -75,10 +75,12 @@ reporting schema is wiped and replaced with the results.
 DESC
     method_option 'force', :type => 'boolean',
       :desc => 'Copy the working schema to production even if there are errors'
+    method_option 'preserve', :type => 'boolean',
+      :desc => 'Do not wipe the working database before beginning ETL (for debugging)'
     def etl
       db = DatabaseInitializer.new(configuration)
       db.set_up_repository(:both)
-      db.replace_schema
+      db.replace_schema unless options[:preserve]
 
       success = TransformLoad.new(configuration).run
       if success || options['force']

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -18,6 +18,7 @@ module NcsNavigator::Warehouse
     autoload :EventStartFromContactTransformer, 'ncs_navigator/warehouse/transformers/event_start_from_contact_transformer'
     autoload :Filters,                 'ncs_navigator/warehouse/transformers/filters'
     autoload :ForeignKeyIndex,         'ncs_navigator/warehouse/transformers/foreign_key_index'
+    autoload :MdesCsv,                 'ncs_navigator/warehouse/transformers/mdes_csv'
     autoload :NoBlankForeignKeysFilter,       'ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter'
     autoload :NoSsuOutreachAllSsusFilter,     'ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter'
     autoload :NoSsuOutreachPlaceholderFilter, 'ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter'

data/lib/ncs_navigator/warehouse/transformers/mdes_csv.rb

@@ -0,0 +1,26 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers
+  module MdesCsv
+    autoload :MultipleTableReader, 'ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader'
+    autoload :TableReader,         'ncs_navigator/warehouse/transformers/mdes_csv/table_reader'
+
+    ##
+    # Creates a transformer that loads a directory full of CSV files,
+    # interpreting each of them using {TableReader}. Files that do not have  the
+    # extension `csv` are ignored; so are subdirectories.
+    #
+    # @return [#transform]
+    def self.from_directory(configuration, directory, options={})
+      readers = Dir["#{directory}/*.csv"].collect do |csv_file|
+        model = File.basename(csv_file).sub(/\.csv$/, '')
+        MdesCsv::TableReader.new(configuration, model, csv_file)
+      end
+
+      EnumTransformer.new(
+        configuration,
+        MdesCsv::MultipleTableReader.new(configuration, readers),
+        options)
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader.rb

@@ -0,0 +1,32 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Transformers::MdesCsv
+  ##
+  # An enumerable that joins the results for several {TableReaders} into a single
+  # {#each} call.
+  class MultipleTableReader
+    include Enumerable
+
+    attr_reader :configuration, :table_readers
+
+    ##
+    # @param [Configuration] configuration
+    # @param [Array<TableReader>] the table readers to concatenate
+    def initialize(configuration, table_readers)
+      @configuration = configuration
+      @table_readers = table_readers.sort_by { |r| configuration.models_module.mdes_order.index(r.model) }
+    end
+
+    def each
+      table_readers.each do |reader|
+        reader.each do |record|
+          yield record
+        end
+      end
+    end
+
+    def name
+      table_readers.collect(&:name).join(', ')
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers/mdes_csv/table_reader.rb

@@ -0,0 +1,64 @@
+require 'ncs_navigator/warehouse'
+
+require 'csv'
+
+module NcsNavigator::Warehouse::Transformers::MdesCsv
+  ##
+  # A streaming reader for a CSV containing one table's-worth of MDES data.
+  #
+  # The CSV is converted into warehouse model instances, one record per row. The
+  # CSV MUST have a header row whose cells indicate the variable contained in
+  # each column.
+  class TableReader
+    include Enumerable
+
+    attr_reader :configuration, :model, :filename
+
+    ##
+    # @param [Configuration] configuration
+    # @param [Class,#to_s] model_designator a name for the expected record type
+    #   from the current CSV. It can be a table name or unqualified warehouse
+    #   model name, or an actual warehouse model class.
+    # @param [String] filename the file containing the CSV
+    def initialize(configuration, model_designator, filename)
+      @configuration = configuration
+      @model =
+        if Class === model_designator
+          model_designator
+        else
+          configuration.model(model_designator.to_s)
+        end
+      @filename = filename
+    end
+
+    def name
+      "#{filename} => #{model.mdes_table_name} table"
+    end
+
+    def each
+      CSV.foreach(filename, :headers => true, :header_converters => [:downcase]) do |row|
+        yield create_instance_for_row(row)
+      end
+    end
+
+    protected
+
+    def create_instance_for_row(row)
+      model.new.tap do |instance|
+        row.each do |header, value|
+          next if header == 'transaction_type'
+
+          setter = "#{header}="
+          unless instance.respond_to?(setter)
+            fail "Unknown attribute #{header.inspect} for #{model}."
+          end
+          instance.send(setter, value)
+        end
+
+        unless instance.key && instance.key.first
+          fail "Missing key (#{model.key.first.name}) for #{model}."
+        end
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.9.0'
+    VERSION = '0.10.0'
   end
 end

data/ncs_mdes_warehouse.gemspec

@@ -18,8 +18,9 @@ Gem::Specification.new do |s|
   s.add_dependency 'ncs_mdes', '~> 0.10'
   s.add_dependency 'ncs_navigator_configuration', '~> 0.2'
 
-  # Post-3.2.4 breaks DataMapper due to https://github.com/rails/rails/pull/6857
-  s.add_dependency 'activesupport', '~> 3.0', '< 3.2.4'
+  # AS 3.2.4-3.2.6 break DataMapper due to https://github.com/rails/rails/pull/6857
+  # TODO: simplify this once we can drop support for AS 3.0 and 3.1
+  s.add_dependency 'activesupport', '~> 3.0', '!= 3.2.4', '!= 3.2.5', '!= 3.2.6'
   s.add_dependency 'i18n', '~> 0.4' # required by activesupport
 
   s.add_dependency 'thor', '~> 0.14.6'
@@ -46,5 +47,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rake', '~> 0.9.2'
   s.add_development_dependency 'yard', '~> 0.7.2'
   s.add_development_dependency 'ci_reporter', '1.6.6'
-  s.add_development_dependency 'fakefs', '~> 0.4.0'
+  s.add_development_dependency 'fakefs', '0.4.0' # FakeFS does not follow semver
 end

data/spec/ncs_navigator/warehouse/models/mdes_model_spec.rb

@@ -1,4 +1,5 @@
 require 'spec_helper'
+require 'nokogiri'
 
 module NcsNavigator::Warehouse::Models
   module Spec
@@ -95,7 +96,10 @@ module NcsNavigator::Warehouse::Models
       it 'always emits a transaction_type entry last since the schema requires it, even though it is meaningless' do
         last = xml.root.elements.last
         last.name.should == 'transaction_type'
-        last['nil'].should == 'true'
+        # Nokogiri using libxml2 2.7.3 vs. libxml2 2.8.0 has different behavior for this test.
+        # With 2.7.3, the attribute is named nil. With 2.8.0, it retains the xsi namespace: 'xsi:nil'.
+        attr_name = last.keys.detect { |a| a =~ /nil$/ }
+        last[attr_name].should == 'true'
       end
 
       it 'emits the columns according to the mdes_order' do

data/spec/ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers::MdesCsv
+  describe MultipleTableReader do
+    def table_reader(model)
+      TableReader.new(spec_config, model, csv_filename(model))
+    end
+
+    def csv_filename(model_name)
+      tmpdir + "#{model_name}.csv"
+    end
+
+    def csv_file(model_name, row_arrays)
+      File.open(csv_filename(model_name), 'w') do |f|
+        row_arrays.each do |row_array|
+          f.puts row_array.join(',')
+        end
+      end
+    end
+
+    let(:table_readers) {
+      [
+        table_reader('LinkPersonParticipant'),
+        table_reader('Person'),
+        table_reader('Participant')
+      ]
+    }
+
+    let(:reader) {
+      MultipleTableReader.new(spec_config, table_readers)
+    }
+
+    let(:sub_enums) { reader.table_readers }
+
+    it 'is Enumerable' do
+      MultipleTableReader.ancestors.should include(Enumerable)
+    end
+
+    describe '#initialize' do
+      it 'orders the sub-enums by MDES order' do
+        sub_enums.collect { |table_reader| table_reader.model.mdes_table_name }.
+          should == %w(person participant link_person_participant)
+      end
+    end
+
+    describe '#each' do
+      it 'yields the results from each table reader in order' do
+        csv_file 'Participant', [
+          %w(p_id), %w(X90)
+        ]
+
+        csv_file 'Person', [
+          %w(person_id), %w(P34)
+        ]
+
+        csv_file 'LinkPersonParticipant', [
+          %w(person_pid_id), %w(T23)
+        ]
+
+        reader.to_a.collect { |rec| [rec.class.mdes_table_name, rec.key.first] }.should == [
+          %w(person P34), %w(participant X90), %w(link_person_participant T23)
+        ]
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/mdes_csv/table_reader_spec.rb

@@ -0,0 +1,118 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers::MdesCsv
+  describe TableReader, :use_mdes do
+    let(:model_designator) { 'person' }
+    let(:filename) { tmpdir + 'people.csv' }
+    let(:reader) { TableReader.new(spec_config, model_designator, filename) }
+
+    it 'is enumerable' do
+      TableReader.ancestors.should include(Enumerable)
+    end
+
+    describe '#initialize' do
+      describe 'model_designator parameter' do
+        it 'accepts a table name to find the model' do
+          TableReader.new(spec_config, 'address', filename).model.
+            should == spec_config.model(:Address)
+        end
+
+        it 'accepts a model name to find the model' do
+          TableReader.new(spec_config, 'Telephone', filename).model.
+            should == spec_config.model(:Telephone)
+        end
+
+        it 'accepts a model class directly' do
+          TableReader.new(spec_config, spec_config.model(:Telephone), filename).model.
+            should == spec_config.model(:Telephone)
+        end
+      end
+    end
+
+    describe '#name' do
+      it 'is the filename and table name' do
+        reader.name.should == "#{filename} => person table"
+      end
+    end
+
+    describe '#each' do
+      let(:records) { reader.to_a }
+      let(:first_record) { records.first }
+
+      def csv_file(row_arrays)
+        File.open(filename, 'w') do |f|
+          row_arrays.each do |row_array|
+            f.puts row_array.join(',')
+          end
+        end
+      end
+
+      it 'produces one record per line' do
+        csv_file [
+          %w(person_id),
+          %w(A3),
+          %w(B7),
+          %w(C8)
+        ]
+
+        records.collect { |rec| rec.person_id }.should == %w(A3 B7 C8)
+      end
+
+      it 'maps record attributes via the headers' do
+        csv_file [
+          %w(last_name first_name person_id),
+          %w(MacMurray Fred FM36)
+        ]
+
+        first_record.person_id.should == 'FM36'
+        first_record.first_name.should == 'Fred'
+        first_record.last_name.should == 'MacMurray'
+      end
+
+      it 'produces model instances' do
+        csv_file [
+          %w(last_name first_name person_id),
+          %w(MacMurray Fred FM36)
+        ]
+
+        first_record.should be_a spec_config.model('person')
+      end
+
+      it 'interprets headers case-insensitively' do
+        csv_file [
+          %w(PERSON_ID FiRST_NaMe),
+          %w(BS34 Barbara)
+        ]
+
+        first_record.first_name.should == 'Barbara'
+      end
+
+      it 'fails for an unknown attribute' do
+        csv_file [
+          %w(person_id helicopter_model),
+          %w(TR900 Huey)
+        ]
+
+        expect { records }.to raise_error(/Unknown attribute "helicopter_model" for NcsNavigator::Warehouse::Models::\w+::Person\./)
+      end
+
+      it 'ignores transaction_type' do
+        csv_file [
+          %w(person_id transaction_type),
+          %w(TR900 NA)
+        ]
+
+        expect { records }.to_not raise_error
+      end
+
+      it 'fails if the key is not provided' do
+        csv_file [
+          %w(first_name last_name),
+          %w(Medford Man)
+        ]
+
+        expect { records }.to raise_error(/Missing key \(person_id\) for NcsNavigator::Warehouse::Models::\w+::Person\./)
+      end
+    end
+  end
+end

data/spec/ncs_navigator/warehouse/transformers/mdes_csv_spec.rb

@@ -0,0 +1,55 @@
+require 'spec_helper'
+
+module NcsNavigator::Warehouse::Transformers
+  describe MdesCsv, :use_mdes do
+    let(:options) { { :filters => lambda { |x| x } } }
+
+    shared_examples 'an MDES CSV transformer' do
+      it 'returns a transformer' do
+        transformer.should respond_to(:transform)
+      end
+
+      it 'passes the options to the transformer' do
+        transformer.filters.to_a.size.should == 1
+      end
+    end
+
+    describe '.from_directory' do
+      let(:dir) { tmpdir('mdes_csv') }
+      let(:transformer) { MdesCsv.from_directory(spec_config, dir.to_s, options) }
+
+      before do
+        %w(
+          person.csv
+          contact.csv
+          event.csv
+          readme.txt
+        ).each do |fn|
+          FileUtils.touch(dir + fn)
+        end
+      end
+
+      include_examples 'an MDES CSV transformer'
+
+      it 'uses a MultipleTableReader' do
+        transformer.enum.should be_a(MdesCsv::MultipleTableReader)
+      end
+
+      it 'uses all the *.csv files in the directory' do
+        transformer.enum.table_readers.collect(&:filename).should == [
+          (dir + 'person.csv').to_s,
+          (dir + 'event.csv').to_s,
+          (dir + 'contact.csv').to_s
+        ]
+      end
+
+      it 'uses the file name (sans ext) as the model designator' do
+        transformer.enum.table_readers.collect(&:model).should == [
+          spec_config.model('person'),
+          spec_config.model('event'),
+          spec_config.model('contact'),
+        ]
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ncs_mdes_warehouse
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.10.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-05 00:00:00.000000000 Z
+date: 2012-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ncs_mdes
@@ -51,9 +51,15 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 3.2.4
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 3.2.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -62,9 +68,15 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '3.0'
-    - - <
+    - - ! '!='
       - !ruby/object:Gem::Version
         version: 3.2.4
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 3.2.5
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 3.2.6
 - !ruby/object:Gem::Dependency
   name: i18n
   requirement: !ruby/object:Gem::Requirement
@@ -422,7 +434,7 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.4.0
   type: :development
@@ -430,7 +442,7 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - '='
       - !ruby/object:Gem::Version
         version: 0.4.0
 description: 
@@ -2313,6 +2325,9 @@ files:
 - lib/ncs_navigator/warehouse/transformers/foreign_key_index.rb
 - lib/ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider.rb
 - lib/ncs_navigator/warehouse/transformers/foreign_key_index/static_key_provider.rb
+- lib/ncs_navigator/warehouse/transformers/mdes_csv.rb
+- lib/ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader.rb
+- lib/ncs_navigator/warehouse/transformers/mdes_csv/table_reader.rb
 - lib/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter.rb
 - lib/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter.rb
 - lib/ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter.rb
@@ -2351,6 +2366,9 @@ files:
 - spec/ncs_navigator/warehouse/transformers/filters_spec.rb
 - spec/ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider_spec.rb
 - spec/ncs_navigator/warehouse/transformers/foreign_key_index_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv/table_reader_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter_spec.rb
@@ -2381,7 +2399,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1614550889051803335
+      hash: -1567355274626065010
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -2390,7 +2408,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1614550889051803335
+      hash: -1567355274626065010
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24
@@ -2423,6 +2441,9 @@ test_files:
 - spec/ncs_navigator/warehouse/transformers/filters_spec.rb
 - spec/ncs_navigator/warehouse/transformers/foreign_key_index/database_key_provider_spec.rb
 - spec/ncs_navigator/warehouse/transformers/foreign_key_index_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv/multiple_table_reader_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv/table_reader_spec.rb
+- spec/ncs_navigator/warehouse/transformers/mdes_csv_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter_spec.rb
 - spec/ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter_spec.rb