ncs_mdes_warehouse 0.12.0 → 0.13.0

data/CHANGELOG.md

@@ -1,6 +1,29 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.13.0
+------
+
+- Move all existing filters to new `Filters` module. Any fully-qualified
+  references to those filters will need to be updated. Filter-name-only
+  references made from within a configuration file will still resolve fine.
+  (#3940)
+
+- Add the concept of named "filter sets" to the warehouse configuration. (#3940)
+
+- Support specifying one or more named filter sets to use with `emit-xml`.
+  (#3940)
+
+- Support specifying a default filter set to use with `emit-xml` if none are
+  specified on the command line. (#3940)
+
+- Split out content enumeration from `XmlEmitter`. `Contents` is an `Enumerable`
+  providing memory-efficient access to every record in a set of tables (or all
+  tables). (#3936)
+
+- Add filters for adding and removing prefixes to/from the keys for all records
+  of a particular type. (#3968)
+
 0.12.0
 ------
 

data/lib/ncs_navigator/warehouse.rb

@@ -9,9 +9,11 @@ module NcsNavigator
     autoload :CLI,                 'ncs_navigator/warehouse/cli'
     autoload :Comparator,          'ncs_navigator/warehouse/comparator'
     autoload :Configuration,       'ncs_navigator/warehouse/configuration'
+    autoload :Contents,            'ncs_navigator/warehouse/contents'
     autoload :DataMapper,          'ncs_navigator/warehouse/data_mapper'
     autoload :DatabaseInitializer, 'ncs_navigator/warehouse/database_initializer'
     autoload :Hooks,               'ncs_navigator/warehouse/hooks'
+    autoload :Filters,             'ncs_navigator/warehouse/filters'
     autoload :Models,              'ncs_navigator/warehouse/models'
     autoload :PostgreSQL,          'ncs_navigator/warehouse/postgresql'
     autoload :StringifyTrace,      'ncs_navigator/warehouse/stringify_trace'

data/lib/ncs_navigator/warehouse/cli.rb

@@ -64,11 +64,19 @@ DESC
       :desc => 'The target directory for automatically-named files. (Default is CWD.)'
     method_option 'tables', :type => :string,
       :desc => 'Emit XML for a subset of tables.', :banner => 'TABLE,TABLE,TABLE'
+    method_option 'filters', :type => :string, :banner => 'FILTER_SET,FILTER_SET',
+      :desc => 'Use these named filter sets when producing the XML. Default is set in configuration. Use --no-filters to disable default without providing an alternative.'
     def emit_xml(filename=nil)
       use_database
 
-      XmlEmitter.new(configuration, filename,
-        options.merge(:tables => options[:tables].try(:split, /\s*,\s*/))).emit_xml
+      options[:tables] = options[:tables].try(:split, /\s*,\s*/)
+      # need to prevent the addition of a :filters key entirely in order
+      # to detect --no-filters, which shows up as :filters=>nil.
+      if options[:filters]
+        options[:filters] = options[:filters].split(/\s*,\s*/)
+      end
+
+      XmlEmitter.new(configuration, filename, options).emit_xml
     end
 
     desc 'etl', 'Performs the full extract-transform-load process for this configuration'

data/lib/ncs_navigator/warehouse/configuration.rb

@@ -133,6 +133,86 @@ module NcsNavigator::Warehouse
       end
     end
 
+    ####
+    #### Filters
+    ####
+
+    ##
+    # @return [Hash<Symbol,#call>] an index of named {CompositeFilter}s
+    #   reflecting the configured named filter sets
+    def filter_sets
+      @filter_sets ||= {}
+    end
+
+    ##
+    # Looks up a filter set by name. Errors out if no match found.
+    #
+    # If you need to refer to a named filter set in your configuration file,
+    # use this method instead of {#filter_sets}. E.g.:
+    #
+    #   c.add_transformer SomeDatabase.create_transformer(c, :filters => [c.filter_set(:some_filters)])
+    #
+    # Using this method instead of {#filter_sets} ensures that you will get a
+    # useful error message if you have a typo in your filter name.
+    #
+    # @return [#call] the {CompositeFilter} registered under the given
+    #   name. If none, it raises {Error}.
+    def filter_set(name)
+      filter_sets[name.to_sym] or raise Error, "Unknown filter set #{name.inspect}."
+    end
+
+    ##
+    # Define a named filter set.
+    #
+    # @param [Symbol,#to_sym] name the name for this set.
+    # @param [#call, Array<#call>] one_or_more_filters the filters to use in
+    #   the set
+    # @return [void]
+    def add_filter_set(name, one_or_more_filters)
+      if filter_sets.has_key?(name.to_sym)
+        raise Error, "There is already a filter set named #{name.inspect}."
+      end
+
+      filters = [*one_or_more_filters].collect do |candidate|
+        case candidate
+        when Symbol
+          filter_set(candidate)
+        else
+          candidate
+        end
+      end
+
+      filters.each do |candidate|
+        unless candidate.respond_to?(:call)
+          if candidate.respond_to?(:new)
+            raise Error, "#{candidate.inspect} does not have a call method. Perhaps you meant #{candidate.inspect}.new?"
+          else
+            raise Error, "#{candidate.inspect} does not have a call method."
+          end
+        end
+      end
+
+      filter_sets[name.to_sym] = Filters::CompositeFilter.new(filters)
+    end
+
+    ##
+    # @return [Symbol,nil] the filter set to with the XML emitter if none is
+    #   specified.
+    attr_reader :default_xml_filter_set
+
+    ##
+    # @param [Symbol,#to_sym] name the name of a known filter set to use by
+    #   default in the XML emitter.
+    # @return [void]
+    def default_xml_filter_set=(name)
+      if name
+        filter_set(name) # fails if unknown
+        @default_xml_filter_set = name.to_sym
+      else
+        @default_xml_filter_set = nil
+      end
+    end
+
     ####
     #### MDES version
     ####

data/lib/ncs_navigator/warehouse/configuration/file_evaluator.rb

@@ -28,7 +28,7 @@ class NcsNavigator::Warehouse::Configuration
     end
 
     def self.const_missing(const)
-      [NcsNavigator::Warehouse::Transformers].each do |ns|
+      [NcsNavigator::Warehouse::Transformers, NcsNavigator::Warehouse::Filters].each do |ns|
         if ns.const_defined?(const)
           return ns.const_get(const)
         end

data/lib/ncs_navigator/warehouse/contents.rb

@@ -0,0 +1,77 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  ##
+  # Provides an Enumerable over all of the content for some or all of the
+  # tables in a warehouse instance. Provides streaming results over batches
+  # for large content sets.
+  class Contents
+    include Enumerable
+
+    ##
+    # @return [Array<Class>] the warehouse models whose records will be enumerated
+    #   by this instance.
+    attr_reader :models
+
+    ##
+    # @return [CompositeFilter] the filters in use on this transformer.
+    attr_reader :filters
+
+    ##
+    # @return [Numeric] the maximum number of records to load into memory before
+    #   yielding them to the consumer.
+    attr_reader :block_size
+
+    ##
+    # Create a new {Contents}.
+    #
+    # @param [Configuration] config the configuration for the
+    #   warehouse from which to iterate over records.
+    #
+    # @option options [Array<#call>,#call] :filters a list of
+    #   filters to use for this transformer
+    # @option options [Fixnum] :block-size (5000) the maximum number
+    #   of records to load into memory before yielding them to the consumer.
+    #   Reduce this to reduce the memory load of the emitter. Increasing it
+    #   will probably not improve performance, even if you have sufficient
+    #   memory to load more records.
+    # @option options [Array<#to_s>] :tables (all for current MDES
+    #   version) the tables to include in the iteration.
+    def initialize(config, options={})
+      @configuration = config
+      @record_count = 0
+      @block_size = options[:'block-size'] || 5000
+
+      @models =
+        if options[:tables]
+          options[:tables].collect { |t| t.to_s }.collect { |t|
+            config.models_module.mdes_order.find { |model| model.mdes_table_name == t }
+          }
+        else
+          config.models_module.mdes_order
+        end
+
+      filter_list = options[:filters]
+      @filters = NcsNavigator::Warehouse::Filters::CompositeFilter.new(
+        filter_list ? [*filter_list].compact : [])
+    end
+
+    ##
+    # Yields each instance of every configured {model #models} in turn.
+    def each
+      models.each do |model|
+        key = model.key.first.name.to_sym
+        count = model.count
+        offset = 0
+        while offset < count
+          model.all(:limit => block_size, :offset => offset, :order => key.asc).each do |instance|
+            filters.call([instance]).each do |filtered_record|
+              yield filtered_record
+            end
+          end
+          offset += block_size
+        end
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/filters.rb

@@ -0,0 +1,19 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse
+  ##
+  # Record filters and related.
+  module Filters
+    autoload :CompositeFilter, 'ncs_navigator/warehouse/filters/composite_filter'
+
+    autoload :AddIdPrefixFilter,              'ncs_navigator/warehouse/filters/add_id_prefix_filter'
+    autoload :ApplyGlobalValuesFilter,        'ncs_navigator/warehouse/filters/apply_global_values_filter'
+    autoload :CodedAsMissingFilter,           'ncs_navigator/warehouse/filters/coded_as_missing_filter'
+    autoload :NoBlankForeignKeysFilter,       'ncs_navigator/warehouse/filters/no_blank_foreign_keys_filter'
+    autoload :NoSsuOutreachAllSsusFilter,     'ncs_navigator/warehouse/filters/no_ssu_outreach_all_ssus_filter'
+    autoload :NoSsuOutreachPlaceholderFilter, 'ncs_navigator/warehouse/filters/no_ssu_outreach_placeholder_filter'
+    autoload :RemoveIdPrefixFilter,           'ncs_navigator/warehouse/filters/remove_id_prefix_filter'
+
+    autoload :RecordIdChangingFilterSupport,  'ncs_navigator/warehouse/filters/record_id_changing_filter_support'
+  end
+end

data/lib/ncs_navigator/warehouse/filters/add_id_prefix_filter.rb

@@ -0,0 +1,32 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Filters
+  ##
+  # A {CompositeFilter filter} which adds a prefix to the ID for every record of
+  # a given type. This filter is stateful and so must be instantiated.
+  class AddIdPrefixFilter
+    include RecordIdChangingFilterSupport
+
+    ##
+    # Creates the filter.
+    #
+    # In addition to the options specified here, this constructor accepts the
+    # options defined on {RecordIdChangingFilterSupport#initialize}.
+    #
+    # @param [Configuration] configuration the warehouse configuration.
+    # @param [Hash] options
+    #
+    # @option options [String] :prefix the prefix to apply.
+    def initialize(configuration, options={})
+      super
+      @prefix = options[:prefix] or fail 'Please specify a :prefix.'
+    end
+
+    ##
+    # @param [String] original_id the incoming ID.
+    # @return [String] the ID with the prefix applied.
+    def changed_id(original_id)
+      "#{@prefix}#{original_id}" if original_id
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/{transformers → filters}/apply_global_values_filter.rb

@@ -1,6 +1,6 @@
 require 'ncs_navigator/warehouse'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   ##
   # A filter which sets certain variables to the same values on all
   # records if those variables are not already set when the filter

data/lib/ncs_navigator/warehouse/{transformers → filters}/coded_as_missing_filter.rb

@@ -1,6 +1,6 @@
 require 'ncs_navigator/warehouse'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   ##
   # Some source data sets include unnecessary placeholder records and
   # values. This filter excludes the following variables and records:

data/lib/ncs_navigator/warehouse/filters/composite_filter.rb

@@ -0,0 +1,77 @@
+require 'forwardable'
+
+module NcsNavigator::Warehouse::Filters
+  ##
+  # Encapsulates an ordered list of filters that can be applied to one or more
+  # records. Filters of this type are used in {EnumTransformer} and {Contents}.
+  #
+  # Each filter is an object with an `call` method. The `call` method takes an
+  # array of zero or more records as input and returns an array of zero or more
+  # records. The input and output arrays may be the same or different. A filter
+  # may add, remove, or mutate records (or all three).
+  #
+  # A filter will be called multiple times for a single transform run, but it
+  # will never see the same record twice. To put it another way, the first
+  # filter will be invoked exactly once per record yielded from the underlying
+  # enumeration. Subsequent filters are invoked on the return value from the
+  # previous filter.
+  #
+  # Depending on the filter order, a particular filter may never see some of the
+  # eventually transformed records. This will happen if records are created by a
+  # filter lower in the filter chain.
+  #
+  # Leaky abstraction note: if a filter needs to change the primary key for a
+  # record, there is unfortunate DataMapper behavior to contend with. DM
+  # memoizes the result of `Resource#key` after the first time it is invoked.
+  # The warehouse infrastructure will certainly have invoked `#key` on a record
+  # which is passed to a filter's call method. In order so that subsequent calls
+  # to `#key` reflect the filter's changes, it needs to work around this. Two
+  # possibilities:
+  #
+  # * Instead of changing the key on the passed-in record, create a new record
+  #   with the new key (and all the other attributes) and return that.
+  # * `record.instance_eval { remove_instance_variable(:@_key) }`
+  class CompositeFilter
+    include Enumerable
+    extend Forwardable
+
+    def_delegators :@filters, :each
+
+    attr_accessor :filters
+
+    def initialize(filter_objects)
+      check_filters(filter_objects)
+
+      @filters = filter_objects
+    end
+
+    ##
+    # @param records [Array] zero or more records to filter.
+    # @return [Array] the filtered records.
+    def call(records)
+      filters.inject(ensure_array(records)) { |result, filter| ensure_array(filter.call(result)) }
+    end
+
+    private
+
+    def check_filters(candidates)
+      candidates.each_with_index do |f, i|
+        unless f.respond_to?(:call)
+          fail "Filter #{i} (#{f.class}) does not have a call method"
+        end
+      end
+    end
+
+    def ensure_array(records)
+      case records
+      when nil
+        []
+      when Enumerable
+        records.to_a
+      else
+        [records]
+      end
+    end
+  end
+end
+

data/lib/ncs_navigator/warehouse/{transformers → filters}/no_blank_foreign_keys_filter.rb

@@ -1,6 +1,6 @@
 require 'ncs_navigator/warehouse'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   ##
   # This filter transforms any blank foreign keys into nil. Blank
   # foreign keys are a common issue in source XML data.

data/lib/ncs_navigator/warehouse/{transformers → filters}/no_ssu_outreach_all_ssus_filter.rb

@@ -1,6 +1,6 @@
 require 'ncs_navigator/warehouse'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   ##
   # A filter which duplicates outreach event records that don't have
   # an SSU ID across all of a center's SSUs.

data/lib/ncs_navigator/warehouse/{transformers → filters}/no_ssu_outreach_placeholder_filter.rb

@@ -1,6 +1,6 @@
 require 'ncs_navigator/warehouse'
 
-module NcsNavigator::Warehouse::Transformers
+module NcsNavigator::Warehouse::Filters
   ##
   # A filter which associates outreach event records that don't have
   # an SSU ID with an automatically-created placeholder SSU.

data/lib/ncs_navigator/warehouse/filters/record_id_changing_filter_support.rb

@@ -0,0 +1,71 @@
+require 'ncs_navigator/warehouse'
+
+module NcsNavigator::Warehouse::Filters
+  ##
+  # A framework for a filter which modifies all ID values for a particular
+  # record type. The class into which it is mixed must implement `changed_id`,
+  # a method which takes an incoming ID value and provides the replacement
+  # value.
+  module RecordIdChangingFilterSupport
+    ##
+    # @return [Class] the warehouse model for which the ID will be prefixed.
+    attr_reader :model
+
+    ##
+    # An inheritable constructor for filters which mix in this module.
+    #
+    # @param [Configuration] configuration the warehouse configuration
+    # @param [Hash] options
+    # @option options [String,Symbol] :table the name of the table for the
+    #   target record type.
+    # @option options [String,Symbol] :model the unqualified name of the
+    #   warehouse model for the target record type. If both this and `:table`
+    #   are specified, `:table` wins.
+    def initialize(configuration, options={})
+      @model =
+        if options[:table]
+          configuration.model(options[:table])
+        elsif options[:model]
+          configuration.model(options[:model])
+        else
+          fail 'Please specify either :table or :model.'
+        end
+      unless self.respond_to?(:changed_id)
+        fail "#{self.class} does not implement changed_id"
+      end
+    end
+
+    ##
+    # Modifies all IDs for the target record type according to the
+    # consumer-defined `changed_id` method.
+    #
+    # @param [Array<MdesModel>] records the records to review and modify
+    # @return [Array<MdesModel>] the same records.
+    #   Any IDs will be updated in place.
+    def call(records)
+      records.each do |rec|
+        if rec.is_a?(model)
+          change_primary_key(rec)
+          # see the class comment on {CompositeFilter}
+          rec.instance_eval { remove_instance_variable(:@_key) if defined?(@_key) }
+        end
+        change_foreign_keys_if_any(rec)
+      end
+    end
+
+    def change_primary_key(record)
+      key_name = model.key.first.name
+      record[key_name] = changed_id(record[key_name])
+    end
+    protected :change_primary_key
+
+    def change_foreign_keys_if_any(record)
+      record.class.relationships.each do |rel|
+        next unless rel.parent_model == model
+        foreign_key = rel.child_key.first.name
+        record[foreign_key] = changed_id(record[foreign_key])
+      end
+    end
+    protected :change_foreign_keys_if_any
+  end
+end