ncs_mdes_warehouse 0.12.0 → 0.13.0

data/lib/ncs_navigator/warehouse/filters/remove_id_prefix_filter.rb

@@ -0,0 +1,32 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Filters
+  ##
+  # A {CompositeFilter filter} which removes a prefix from the ID for every
+  # record of a given type. This filter is stateful and so must be instantiated.
+  class RemoveIdPrefixFilter
+    include RecordIdChangingFilterSupport
+
+    ##
+    # Creates the filter.
+    #
+    # In addition to the options specified here, this constructor accepts the
+    # options defined on {RecordIdChangingFilterSupport#initialize}.
+    #
+    # @param [Configuration] configuration the warehouse configuration.
+    # @param [Hash] options
+    #
+    # @option options [String] :prefix the prefix to remove.
+    def initialize(configuration, options={})
+      super
+      @prefix = options[:prefix] or fail 'Please specify a :prefix.'
+    end
+
+    ##
+    # @param [String] original_id the incoming ID.
+    # @return [String] the ID with the prefix removed.
+    def changed_id(original_id)
+      original_id.sub(/\A#{@prefix}/, '') if original_id
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/transformers.rb

@@ -11,17 +11,11 @@ module NcsNavigator::Warehouse
   # `Enumerable` of MDES Warehouse model instances, validates and
   # saves them. It's a good general base for your own transformers.
   module Transformers
-    autoload :ApplyGlobalValuesFilter, 'ncs_navigator/warehouse/transformers/apply_global_values_filter'
-    autoload :CodedAsMissingFilter,    'ncs_navigator/warehouse/transformers/coded_as_missing_filter'
     autoload :Database,                'ncs_navigator/warehouse/transformers/database'
     autoload :EnumTransformer,         'ncs_navigator/warehouse/transformers/enum_transformer'
     autoload :EventStartFromContactTransformer, 'ncs_navigator/warehouse/transformers/event_start_from_contact_transformer'
-    autoload :Filters,                 'ncs_navigator/warehouse/transformers/filters'
     autoload :ForeignKeyIndex,         'ncs_navigator/warehouse/transformers/foreign_key_index'
     autoload :MdesCsv,                 'ncs_navigator/warehouse/transformers/mdes_csv'
-    autoload :NoBlankForeignKeysFilter,       'ncs_navigator/warehouse/transformers/no_blank_foreign_keys_filter'
-    autoload :NoSsuOutreachAllSsusFilter,     'ncs_navigator/warehouse/transformers/no_ssu_outreach_all_ssus_filter'
-    autoload :NoSsuOutreachPlaceholderFilter, 'ncs_navigator/warehouse/transformers/no_ssu_outreach_placeholder_filter'
     autoload :SamplingUnits,           'ncs_navigator/warehouse/transformers/sampling_units'
     autoload :SqlTransformer,          'ncs_navigator/warehouse/transformers/sql_transformer'
     autoload :SubprocessTransformer,   'ncs_navigator/warehouse/transformers/subprocess_transformer'

data/lib/ncs_navigator/warehouse/transformers/database.rb

@@ -13,14 +13,11 @@ module NcsNavigator::Warehouse::Transformers
   #     class StaffPortalTransformer
   #       include NcsNavigator::Warehouse::Transformers::Database
   #
-  #       # Include the models for the version of the MDES this
-  #       # transformer is compatible with
-  #       include NcsNavigator::Warehouse::Models::TwoPointZero
-  #
   #       bcdatabase :name => 'ncs_staff_portal'
   #
-  #       produce_records(:staff) do |row|
-  #         Staff.new(
+  #       produce_records(:staff) do |row, meta|
+  #         staff_model = meta[:configuration].model(:staff)
+  #         staff_model.new(
   #           :staff_id => 'SP' + row.username
   #           # etc.
   #         )
@@ -32,8 +29,9 @@ module NcsNavigator::Warehouse::Transformers
   #           SELECT sub.*, s.username
   #           FROM staff_languages sub INNER JOIN staff s ON sub.staff_id=s.id
   #         )
-  #       ) do |row|
-  #         StaffLanguages.new(
+  #       ) do |row, meta|
+  #         sl_model = meta[:configuration].model(:staff_language)
+  #         sl_model.new(
   #           :staff_language_id => 'SP' + row.id,
   #           :staff_id => 'SP' + row.username,
   #           :staff_lang => row.lang_code
@@ -232,6 +230,14 @@ module NcsNavigator::Warehouse::Transformers
       # Define a translation from the results of a query into one or
       # more warehouse records.
       #
+      # The optional second parameter the `logic` proc/block is a hash of
+      # metadata. That metadata contains a single key:
+      #
+      # * :configuration. Provides the {Configuration} in use by the warehouse
+      #   that is executing this transformer. Among other things, this allows
+      #   {dynamic lookup of models Configuration#model} based on the in-use
+      #   MDES version.
+      #
       # @param [Symbol] name the name of this producer; if you don't
       #   specify a `:query`, the default is to return every row from
       #   the application table with this name.

data/lib/ncs_navigator/warehouse/transformers/enum_transformer.rb

@@ -1,4 +1,5 @@
 require 'ncs_navigator/warehouse'
+require 'ncs_navigator/warehouse/data_mapper_patches'
 
 require 'forwardable'
 
@@ -32,7 +33,7 @@ module NcsNavigator::Warehouse::Transformers
     attr_reader :enum
 
     ##
-    # @return [Filters] the filters in use on this transformer.
+    # @return [CompositeFilter] the filters in use on this transformer.
     attr_reader :filters
 
     ##
@@ -54,12 +55,13 @@ module NcsNavigator::Warehouse::Transformers
     #   `:ignore` means do not attempt to save the duplicate.
     #   `:replace` means substitute the duplicate for the existing record.
     #
-    # @see Filters
+    # @see CompositeFilter
     def initialize(configuration, enum, options={})
       @configuration = configuration
       @enum = enum
       filter_list = options.delete(:filters)
-      @filters = Filters.new(filter_list ? [*filter_list].compact : [])
+      @filters = NcsNavigator::Warehouse::Filters::CompositeFilter.new(
+        filter_list ? [*filter_list].compact : [])
       @duplicates = options.delete(:duplicates) || :error
       @duplicates_strategy = select_duplicates_strategy
 

data/lib/ncs_navigator/warehouse/version.rb

@@ -1,5 +1,5 @@
 module NcsNavigator
   module Warehouse
-    VERSION = '0.12.0'
+    VERSION = '0.13.0'
   end
 end

data/lib/ncs_navigator/warehouse/xml_emitter.rb

@@ -88,34 +88,56 @@ XML
     # @param [Pathname,#to_s,nil] filename the filename to which the output
     #   will be written. If `nil`, the {.default_filename} is used.
     #
+    # @option options [Boolean] :include-pii (false) should PII
+    #   variable values be included in the XML?
+    # @option options [Boolean] :zip (true) should a ZIP file be
+    #   produced alongside the XML file?
+    # @option options [Enumerable] :content an enumerable over the records to
+    #   emit. A {Contents} will be created using the other options
+    #   if this is not specified.
     # @option options [Fixnum] :block-size (5000) the maximum number
     #   of records to load into memory before writing them to the XML
     #   file. Reduce this to reduce the memory load of the emitter.
     #   Increasing it will probably not improve performance, even if
-    #   you have sufficient memory to load more records.
-    # @option options [Boolean] :include-pii (false) should PII
-    #   variable values be included in the XML?
+    #   you have sufficient memory to load more records. Only used if `:content`
+    #   is not specified.
     # @option options [Array<#to_s>] :tables (all for current MDES
-    #   version) the tables to include in the emitted XML.
-    # @option options [Boolean] :zip (true) should a ZIP file be
-    #   produced alongside the XML file?
+    #   version) the tables to include in the emitted XML. Only used if
+    #   `:content` is not specified.
+    # @option options [Array<#to_sym>,nil] :filters (config.default_xml_filter_set)
+    #    named filter sets to apply to the data before emitting XML. Only used if
+    #   `:content` is not specified. If the option is not specified at all, the
+    #   {default in the configuration Configuration#default_xml_filter_set} will
+    #   be used, if any. To avoid using even this default, include
+    #   `:filters=>nil` in the options.
     def initialize(config, filename, options={})
       @configuration = config
-      @record_count = 0
-      @block_size = options[:'block-size'] || 5000
       @zip = options.has_key?(:zip) ? options[:zip] : true
 
       @xml_files = determine_files_to_create(filename, options)
 
+      @tracker = ProgressTracker.new(@configuration)
 
-      @models =
-        if options[:tables]
-          options[:tables].collect { |t| t.to_s }.collect { |t|
-            config.models_module.mdes_order.find { |model| model.mdes_table_name == t }
-          }
-        else
-          config.models_module.mdes_order
-        end
+      if options[:content]
+        @content_enumerator = options[:content]
+      else
+        filter_names =
+          if options.has_key?(:filters)
+            options[:filters] ? options[:filters] : []
+          else
+            [config.default_xml_filter_set].compact
+          end
+        filters =
+          unless filter_names.empty?
+            filter_names.collect { |n| config.filter_set(n) }
+          end
+
+        @content_enumerator = Contents.new(config, {
+          :tables => options[:tables],
+          :'block-size' => options[:'block-size'],
+          :filters => filters
+        })
+      end
     end
 
     ##
@@ -126,19 +148,16 @@ XML
       shell.say_line("Exporting to #{xml_files.collect(&:describe).join(', ')}")
       log.info("Beginning XML export to #{xml_files.collect(&:describe).join(', ')}")
 
-      @start = Time.now
+      @tracker.start!
       xml_files.each { |xf| xf.write HEADER_TEMPLATE.result(binding) }
-      models.each do |model|
-        shell.clear_line_then_say('Writing XML for %33s' % model.mdes_table_name)
-        write_all_xml_for_model(model)
+      @content_enumerator.each do |instance|
+        @tracker.starting_instance(instance)
+        xml_files.each { |xf| xf.write_instance(instance) }
+        @tracker.finish_instance(instance)
       end
       xml_files.each { |xf| xf.write FOOTER_TEMPLATE }
       xml_files.each { |xf| xf.close }
-      @end = Time.now
-
-      msg = "%d records written in %d seconds (%.1f/sec).\n" % [@record_count, emit_time, emit_rate]
-      shell.clear_line_then_say(msg)
-      log.info(msg)
+      @tracker.stop!
 
       xml_files.each { |xf| xf.zip_if_desired }
       log.info("XML export complete")
@@ -217,23 +236,6 @@ XML
       end
     end
 
-    def write_all_xml_for_model(model)
-      shell.say(' %20s' % '[loading]')
-      key = model.key.first.name.to_sym
-      count = model.count
-      offset = 0
-      while offset < count
-        shell.back_up_and_say(20, '%20s' % '[loading]')
-        model.all(:limit => @block_size, :offset => offset, :order => key.asc).each do |instance|
-          xml_files.each { |xf| xf.write_instance(instance) }
-          @record_count += 1
-
-          shell.back_up_and_say(20, '%5d (%5.1f/sec)' % [@record_count, emit_rate])
-        end
-        offset += @block_size
-      end
-    end
-
     def sc_id
       configuration.navigator.sc_id
     end
@@ -246,14 +248,6 @@ XML
       configuration.mdes.specification_version
     end
 
-    def emit_time
-      (@end || Time.now) - @start
-    end
-
-    def emit_rate
-      @record_count / emit_time
-    end
-
     ##
     # @private
     #
@@ -300,5 +294,62 @@ XML
         end
       end
     end
+
+    ##
+    # @private
+    class ProgressTracker
+      extend Forwardable
+
+      attr_reader :start, :stop, :record_count, :shell
+
+      def_delegators :@configuration, :shell, :log
+
+      def initialize(config)
+        @record_count = 0
+        @current_model = nil
+        @configuration = config
+      end
+
+      def start!
+        @start = Time.now
+      end
+
+      def starting_instance(i)
+        model_changed!(i.class) unless i.class == @current_model
+      end
+
+      def model_changed!(new_model)
+        shell.clear_line_then_say('Writing XML for %33s' % new_model.mdes_table_name)
+        shell.say(' %20s' % '[loading]')
+        @current_model = new_model
+      end
+      private :model_changed!
+
+      def finish_instance(i)
+        @record_count += 1
+        update_status
+      end
+
+      def update_status
+        shell.back_up_and_say(20, '%5d (%5.1f/sec)' % [record_count, emit_rate])
+      end
+      private :update_status
+
+      def stop!
+        @stop = Time.now
+
+        msg = "%d records written in %d seconds (%.1f/sec).\n" % [record_count, emit_time, emit_rate]
+        shell.clear_line_then_say(msg)
+        log.info(msg)
+      end
+
+      def emit_time
+        (stop || Time.now) - start
+      end
+
+      def emit_rate
+        record_count / emit_time
+      end
+    end
   end
 end

data/sample_configuration.rb

@@ -46,3 +46,15 @@ c.add_transformer Bar.new(configuration)
 # :normal or :quiet. The default is :normal; it will usually make more
 # sense to control this from the command line.
 #c.output_level = :normal
+
+# Give a set of one or more filters a name.
+# This name can be used with `emit-xml` to apply the filter during export.
+c.add_filter_set :quux, [FilterOne, lambda { |recs| recs }, FilterThree.new(c)]
+
+# Filter sets may include other filter sets:
+c.add_filter_set :baz, [FilterSeven.new, :quux]
+
+# Specify a filter set to use by default in emit-xml. Any filter referenced
+# using --filters in the invocation of emit-xml will be used _instead_ of this
+# filter.
+c.default_xml_filter_set = :quux

data/spec/ncs_navigator/warehouse/configuration_spec.rb

@@ -102,6 +102,100 @@ module NcsNavigator::Warehouse
       end
     end
 
+    describe '#add_filter_set' do
+      let(:a_filter) { lambda { |recs| recs + ['A'] } }
+      let(:b_filter) { lambda { |recs| ['B'] + recs } }
+      let(:c_filter) { lambda { |recs| ['C'] + recs + ['C'] } }
+
+      describe 'with one valid filter' do
+        before do
+          config.add_filter_set(:quux, a_filter)
+        end
+
+        it 'adds the filter set' do
+          config.filter_set(:quux).should_not be_nil
+        end
+
+        it 'produces a callable filter' do
+          config.filter_set(:quux).call(['D']).should == %w(D A)
+        end
+      end
+
+      describe 'with an array valid filters' do
+        before do
+          config.add_filter_set(:some, [b_filter, c_filter, a_filter])
+        end
+
+        it 'adds the filter set' do
+          config.filter_set(:some).should_not be_nil
+        end
+
+        it 'produces a single callable filter' do
+          config.filter_set(:some).call(['D']).should == %w(C B D C A)
+        end
+      end
+
+      describe 'with a reference to another filter set' do
+        it 'expands the reference if the name is known' do
+          config.add_filter_set(:gamma, c_filter)
+          config.add_filter_set(:beta, [:gamma, b_filter])
+
+          config.filter_set(:beta).call(['D']).should == %w(B C D C)
+        end
+
+        it 'fails if the name is unknown' do
+          expect {
+            config.add_filter_set(:beta, [:gamma, b_filter])
+          }.to raise_error('Unknown filter set :gamma.')
+        end
+      end
+
+      describe 'with an object without a #call method' do
+        it 'gives a helpful message if the object is constructable' do
+          lambda { config.add_filter_set(:quux, String) }.
+            should raise_error('String does not have a call method. Perhaps you meant String.new?')
+        end
+
+        it 'gives a helpful message if the object is just an instance' do
+          lambda { config.add_filter_set(:quux, "not a filter") }.
+            should raise_error('"not a filter" does not have a call method.')
+        end
+
+        it 'does not add the filter set' do
+          config.add_filter_set(:quux, 'not a filter') rescue nil
+          config.filter_sets.should_not have_key(:quux)
+        end
+      end
+
+      describe 'when there is a set with the same name' do
+        it 'throws a useful error message' do
+          config.add_filter_set(:beta, b_filter)
+          expect { config.add_filter_set(:beta, c_filter) }.
+            to raise_error('There is already a filter set named :beta.')
+        end
+      end
+    end
+
+    describe '#default_xml_filter_set' do
+      it 'defaults to nil' do
+        config.default_xml_filter_set.should be_nil
+      end
+    end
+
+    describe '#default_xml_filter_set=' do
+      it 'accepts a known filter set name' do
+        config.add_filter_set :foo, lambda { |recs| recs }
+        config.default_xml_filter_set = :foo
+
+        config.default_xml_filter_set.should == :foo
+      end
+
+      it 'fails with an unknown filter set name' do
+        expect { config.default_xml_filter_set = :foo }.
+          to raise_error("Unknown filter set :foo.")
+      end
+    end
+
     describe '#mdes_version=' do
       context 'for a known version', :slow, :use_mdes, :modifies_warehouse_state  do
         it 'makes the models available' do
@@ -542,6 +636,14 @@ module NcsNavigator::Warehouse
         subject.log_file.to_s.should == 'NcsNavigator::Warehouse::Transformers::EnumTransformer.log'
       end
 
+      it 'evaluates constants from NcsNavigator::Warehouse::Filters' do
+        write_file do |f|
+          f.puts 'c.log_file = "#{CompositeFilter}.log"'
+        end
+
+        subject.log_file.to_s.should == 'NcsNavigator::Warehouse::Filters::CompositeFilter.log'
+      end
+
       it 'reports missing constants as bare' do
         write_file do |f|
           f.puts 'c.add_transformer = ATransformerIForgotToRequire'