ncs_mdes_warehouse 0.1.1 → 0.2.0

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format=nested

data/CHANGELOG.md

@@ -1,6 +1,26 @@
 NCS Navigator MDES Warehouse History
 ====================================
 
+0.2.0
+-----
+
+- Automatically set PSU ID and recruitment type in
+  `Transformers::EnumTransformer` if they are not already set. (#1630,
+  #1648)
+
+- ETL: store invalid or failing record IDs as a separate column
+  instead of as part of TransformError#message. (#1636)
+
+- When an MDES model string variable is set from a BigDecimal, coerce
+  it to a floating point string. The ruby default is scientific
+  notation, but that's not permitted in the MDES.
+
+- New options for `emit-xml`: `--tables`, `--zip`,
+  `--include-pii`. (#1612, #1657, #1658)
+
+- Add `compare` subcommand in `mdes-wh` for comparing the contents of
+  two warehouses. (#1667)
+
 0.1.1
 -----
 

data/Rakefile

@@ -15,12 +15,11 @@ task :spec => 'spec:all'
 namespace :spec do
   RSpec::Core::RakeTask.new(:fast) do |t|
     t.pattern = "spec/**/*_spec.rb"
-    t.rspec_opts = %q(--format nested --tag ~slow)
+    t.rspec_opts = %q(--tag ~slow)
   end
 
   RSpec::Core::RakeTask.new(:all) do |t|
     t.pattern = "spec/**/*_spec.rb"
-    t.rspec_opts = %q(--format nested)
   end
 end
 
@@ -38,7 +37,7 @@ namespace :ci do
 
   task :spec_setup do
     ENV['CI_REPORTS'] = 'reports/spec-xml'
-    ENV['SPEC_OPTS'] = "#{ENV['SPEC_OPTS']} --format nested"
+    ENV['SPEC_OPTS'] = "#{ENV['SPEC_OPTS']} --format nested --no-color"
   end
 
   task :spec => [:spec_setup, 'ci:setup:rspecbase', 'rake:spec']

data/generated_models/ncs_navigator/warehouse/models/two_point_zero/spec_blood.rb

@@ -124,9 +124,9 @@ module NcsNavigator::Warehouse::Models::TwoPointZero
     property   :centrifuge_location,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :required => true, :length => 1..2, :set => ["1", "2", "-3", "-4"] }
-    belongs_to :equip,
-               'NcsNavigator::Warehouse::Models::TwoPointZero::SpecEquipment',
-               :child_key => [ :equip_id ], :required => false
+    property   :equip_id,
+               NcsNavigator::Warehouse::DataMapper::NcsString,
+               { :length => 0..36 }
     property   :centrifuge_time,
                NcsNavigator::Warehouse::DataMapper::NcsString,
                { :length => 0..5, :format => /^([0-9][0-9]:[0-9][0-9])?$/ }

data/lib/ncs_navigator/warehouse.rb

@@ -7,6 +7,7 @@ module NcsNavigator
     DEFAULT_MDES_VERSION = '2.0'
 
     autoload :CLI,                 'ncs_navigator/warehouse/cli'
+    autoload :Comparator,          'ncs_navigator/warehouse/comparator'
     autoload :Configuration,       'ncs_navigator/warehouse/configuration'
     autoload :DataMapper,          'ncs_navigator/warehouse/data_mapper'
     autoload :DatabaseInitializer, 'ncs_navigator/warehouse/database_initializer'

data/lib/ncs_navigator/warehouse/cli.rb

@@ -54,10 +54,20 @@ the contents of the current reporting database. The default name for
 the XML file is the county name for the PSU plus the date; e.g.,
 cook-20110728.xml.
 DESC
+    method_option 'block-size', :type => :numeric, :aliases => %w(-b),
+      :desc => 'The maximum number of records to have in memory at once.',
+      :default => 5000
+    method_option 'include-pii', :type => :boolean, :default => false,
+      :desc => 'Include PII values in the emitted XML.'
+    method_option 'zip', :type => :boolean, :default => true,
+      :desc => 'Create a zip file alongside the XML. (Use --no-zip to disable.)'
+    method_option 'tables', :type => :string,
+      :desc => 'Emit XML for a subset of tables.', :banner => 'TABLE,TABLE,TABLE'
     def emit_xml(filename=nil)
       use_database
 
-      XmlEmitter.new(configuration, filename).emit_xml
+      XmlEmitter.new(configuration, filename,
+        options.merge(:tables => options[:tables].try(:split, /\s*,\s*/))).emit_xml
     end
 
     desc 'etl', 'Performs the full extract-transform-load process for this configuration'
@@ -83,6 +93,41 @@ DESC
         exit 1
       end
     end
+
+    desc 'compare', 'Compares the contents of two warehouses, A & B.'
+    long_desc <<-DESC
+Compares the contents of the MDES tables in two warehouses.
+
+The comparison can be done at three levels:
+
+1. Record counts.
+
+2. IDs. (Which record IDs appear in one warehouse and not the other?)
+
+3. Full contents. (Matching records based on ID, what variables have different values?)
+
+Each level includes the one before it (i.e., doing a full content
+comparison also does ID and count comparisons). Higher levels skip
+tables which would add heat without light; i.e., level 2 only compares
+IDs for a table when there are at least some records in each warehouse
+and level 3 only compares content for tables where there are some
+overlapping IDs.
+DESC
+    method_option 'warehouse-a', :type => :string, :aliases => %w(-a),
+      :desc => 'The configuration file for warehouse A. The environment default will be used if not specified.'
+    method_option 'warehouse-b', :type => :string, :required => true, :aliases => %w(-b),
+      :desc => 'The configuration file for warehouse B.'
+    method_option 'level', :type => :numeric, :default => 1,
+      :desc => 'The level of detail for the comparison.'
+    def compare
+      if options['warehouse-a']
+        options['config'] = options['warehouse-a']
+      end
+      config_a = configuration
+      config_b = Configuration.from_file(options['warehouse-b'])
+
+      Comparator.new(config_a, config_b, options).compare
+    end
   end
 end
 

data/lib/ncs_navigator/warehouse/comparator.rb

@@ -0,0 +1,257 @@
+require 'ncs_navigator/warehouse'
+
+require 'forwardable'
+
+module NcsNavigator::Warehouse
+  class Comparator
+    extend Forwardable
+
+    attr_reader :level
+    attr_reader :a, :b
+
+    def_delegators :a, :shell
+
+    def initialize(config_a, config_b, options={})
+      @a = config_a; @b = config_b
+      @level = options[:level] ||= 1
+
+      shell.clear_line_and_say "Connecting to warehouse A..."
+      DatabaseInitializer.new(@a).set_up_repository(:reporting, 'a')
+      shell.clear_line_and_say "Connecting to warehouse B..."
+      DatabaseInitializer.new(@b).set_up_repository(:reporting, 'b')
+      shell.clear_line_and_say ''
+    end
+
+    def compare
+      shell.say_line "Running level #{level} compare..."
+      count_diff if level >= 1
+      id_diff if level >= 2
+      content_diff if level >= 3
+    end
+
+    def count_diff
+      @count_diffs = collect_differences(CountDiff, models.collect { |m| [m] })
+      shell_summarize_differences(CountDiff, @count_diffs)
+    end
+
+    def id_diff
+      @id_diffs = collect_differences(
+        IdDiff, models.select { |m| no_zero_counts?(m) }.collect { |m| [m] })
+      shell_summarize_differences(IdDiff, @id_diffs)
+    end
+
+    def content_diff
+      @content_diffs = collect_differences(
+        ContentDiff, models.collect { |m| [m, list_shared_ids(m)] }.select { |m, shared| shared })
+      shell_summarize_differences(ContentDiff, @content_diffs)
+    end
+
+    private
+
+    def models
+      @models ||= begin
+                    shell.clear_line_and_say "Loading MDES models..."
+                    a.models_module.mdes_order
+                  end
+    end
+
+    def collect_differences(differ, arg_sets)
+      start = Time.now
+      arg_sets.collect do |arg_set|
+        model = arg_set.first
+        shell.clear_line_and_say "comparing #{differ.thing_plural} for #{model.mdes_table_name}..."
+        differ.new(*arg_set).compare
+      end.compact.tap {
+        shell.clear_line_and_say "#{differ.thing} comparisons complete in %.1fs.\n" % (Time.now - start)
+      }
+    end
+
+    def shell_summarize_differences(differ, diffs)
+      if diffs.empty?
+        puts "There are no #{differ.thing} differences."
+      else
+        puts "#{differ.thing_plural} differ in #{diffs.size} table#{'s' if diffs.size != 1}:"
+        diffs.each do |diff|
+          puts diff.shell_summary
+        end
+      end
+      puts
+    end
+
+    def no_zero_counts?(model)
+      if diff = @count_diffs.find { |d| d.table_name == model.mdes_table_name }
+        diff.a_count != 0 && diff.b_count != 0
+      else
+        true
+      end
+    end
+
+    def list_shared_ids(model)
+      if diff = @id_diffs.find { |d| d.model == model }
+        diff.shared
+      else
+        nil
+      end
+    end
+
+    # @private
+    class CountDiff
+      attr_reader :table_name, :a_count, :b_count
+
+      def self.thing; 'count'; end
+      def self.thing_plural; 'counts'; end
+
+      def initialize(model)
+        @table_name = model.mdes_table_name
+        @q = "SELECT COUNT(*) FROM #{table_name}"
+      end
+
+      def compare
+        @a_count = count('a_reporting')
+        @b_count = count('b_reporting')
+        if difference != 0
+          self
+        end
+      end
+
+      def difference
+        a_count - b_count
+      end
+
+      def count(repo)
+        ::DataMapper.repository(repo).adapter.select(@q).first
+      end
+
+      def shell_summary
+        "  %30s: A = %6d %s %-6d = B | %6d" % [
+          table_name,
+          a_count, a_count > b_count ? '>' : '<', b_count,
+          difference.abs
+        ]
+      end
+    end
+
+    # @private
+    class IdDiff
+      attr_reader :model, :a_ids, :b_ids
+
+      def self.thing; 'ID'; end
+      def self.thing_plural; 'IDs'; end
+
+      def initialize(model)
+        @model = model
+        @q = "SELECT #{model.key.first.name} FROM #{model.mdes_table_name}"
+      end
+
+      def compare
+        @a_ids = ids('a_reporting')
+        @b_ids = ids('b_reporting')
+        unless a_only.empty? && b_only.empty?
+          self
+        end
+      end
+
+      def a_only
+        @a_only ||= a_ids - b_ids
+      end
+
+      def b_only
+        @b_only ||= b_ids - a_ids
+      end
+
+      def shared
+        @shared ||= a_ids & b_ids
+      end
+
+      def ids(repo)
+        ::DataMapper.repository(repo).adapter.select(@q)
+      end
+
+      def shell_summary
+        [
+          "  #{model.mdes_table_name}",
+         ("    #{a_only.size} in A only: #{a_only.join(', ')}" unless a_only.empty?),
+         ("    #{b_only.size} in B only: #{b_only.join(', ')}" unless b_only.empty?),
+          "    #{shared.size} the same in both A and B."
+        ].compact.join("\n")
+      end
+    end
+
+    # @private
+    class ContentDiff
+      BLOCK_SIZE = 5000
+
+      attr_reader :model, :shared_ids
+
+      def self.thing; 'content'; end
+      def self.thing_plural; 'contents'; end
+
+      def initialize(model, shared_ids)
+        @model = model
+        @shared_ids = shared_ids
+      end
+
+      def compare
+        id_blocks.each do |ids|
+          key = model.key.first.name
+          ids_param = "'#{ids.join("', '")}'"
+          find_all = lambda { |repo_name|
+            q = "SELECT * FROM #{model.mdes_table_name} WHERE #{key} IN (#{ids_param}) ORDER BY #{key}"
+            ::DataMapper.repository(repo_name).adapter.select(q)
+          }
+          a_records = find_all['a_reporting']
+          b_records = find_all['b_reporting']
+
+          a_records.zip(b_records).each do |a_rec, b_rec|
+            compare_records(a_rec[key], a_rec, b_rec)
+          end
+        end
+
+        unless differences.empty?
+          self
+        end
+      end
+
+      def differences
+        @differences ||= {}
+      end
+
+      def id_blocks
+        @id_blocks ||=
+          begin
+            allocated = 0
+            blocks = []
+            while allocated < shared_ids.size
+              blocks << shared_ids[allocated...(allocated + BLOCK_SIZE)]
+              allocated += BLOCK_SIZE
+            end
+            blocks
+          end
+      end
+
+      def compare_records(id, a_record, b_record)
+        rec_diff = a_record.members.inject({}) { |h, prop|
+          a_value = a_record.send(prop)
+          b_value = b_record.send(prop)
+          h[prop] = [a_value, b_value] if a_value != b_value
+          h
+        }
+        unless rec_diff.empty?
+          differences[id] = rec_diff
+        end
+      end
+
+      def shell_summary
+        [
+          "  #{model.mdes_table_name}:",
+          differences.each_pair.collect do |id, rec_diff|
+            ["    #{id}:"] +
+              rec_diff.each_pair.collect { |var, vals|
+                "      #{var}: #{vals.collect(&:inspect).join(' != ')}"
+              }
+          end
+        ].flatten.join("\n")
+      end
+    end
+  end
+end

data/lib/ncs_navigator/warehouse/data_mapper.rb

@@ -23,16 +23,36 @@ module NcsNavigator::Warehouse
       end
     end
 
+    module NcsStringType
+      protected
+
+      def self.included(into)
+        into.class_eval do
+          alias :original_typecast_to_primitive :typecast_to_primitive
+
+          def typecast_to_primitive(value)
+            if value.is_a?(BigDecimal)
+              value.to_s('F')
+            else
+              original_typecast_to_primitive(value)
+            end
+          end
+        end
+      end
+    end
+
     ##
     # DataMapper `:string` that's an {NcsType}.
     class NcsString < ::DataMapper::Property::String
       include NcsType
+      include NcsStringType
     end
 
     ##
     # DataMapper `:text` that's an {NcsType}.
     class NcsText < ::DataMapper::Property::Text
       include NcsType
+      include NcsStringType
     end
 
     ##

data/lib/ncs_navigator/warehouse/database_initializer.rb

@@ -44,18 +44,17 @@ module NcsNavigator::Warehouse
     #   be the reporting repo unless only the working repo is
     #   configured in.
     # @return [void]
-    def set_up_repository(mode=:reporting)
+    def set_up_repository(mode=:reporting, prefix="mdes_warehouse")
       fail "Invalid mode #{mode.inspect}" unless [:reporting, :working, :both].include?(mode)
       modes = case mode
               when :both then [:reporting, :working]
               else [mode]
               end
       connect_one(modes.first, :default)
-      modes.each { |m| connect_one(m) }
+      modes.each { |m| connect_one(m, [prefix, m].join('_').to_sym) }
     end
 
-    def connect_one(which_one, dm_name=nil)
-      dm_name ||= :"mdes_warehouse_#{which_one}"
+    def connect_one(which_one, dm_name)
       log.info "Connecting DataMapper repository #{dm_name.inspect}"
       p = params(which_one)
       log.debug "  using #{p.merge('password' => 'SUPPRESSED').inspect}"