terrapop_models 0.1.0

data/app/models/sample_design.rb

@@ -0,0 +1,300 @@
+require 'csv'
+
+####
+#
+# To get the _geog_setup.yml files, run things like this:
+#
+#  Country.where({}).each{|country| SampleDesign.to_geog_setup(country) }
+#
+
+class SampleDesign < ActiveRecord::Base
+
+  belongs_to :country
+
+  # Parses a sample design file (which is tab-delimited CSV) for a specified sample, returning
+  # the metadata in a hashtable.  We use the CSV library as the parsed files use some of the
+  # more obscure features of CSVs, such as multiline values.
+  
+  @@has_startup = false
+  
+  def self.startup
+    
+    unless @@has_startup
+      @tag_mapping = {
+          'sample_id' => [''.upcase],
+          'label' => ['custom:label'.upcase],
+          'description' => ['census characteristics:title'.upcase],
+          'local_title' => [''.upcase],
+          'is_weighted' => [''.upcase],
+          'census_agency' => ['census characteristics:census agency'.upcase],
+          'population_universe' => ['census characteristics:population universe'.upcase],
+          'de_jure_or_de_facto' => ['census characteristics:de jure or de facto'.upcase],
+          'enumeration_unit' => ['census characteristics:enumeration unit'.upcase],
+          'census_start_date' => ['census characteristics:census day'.upcase],
+          'census_end_date' => [''.upcase],
+          'fieldwork_start_period' => ['census characteristics:census day'.upcase],
+          'fieldwork_end_period' => [''.upcase],
+          'enumeration_forms' => ['census characteristics:enumeration forms'.upcase],
+          'type_of_fieldwork' => ['census characteristics:type of fieldwork'.upcase],
+          'respondent' => ['census characteristics:respondent'.upcase],
+          'coverage' => ['census characteristics:coverage'.upcase],
+          'undercount' => ['census characteristics:undercount'.upcase],
+          'undercount_notes' => [''.upcase],
+
+          'microdata_source' => ['microdata sample characteristics:microdata source'.upcase],
+          'long_form_sample_design' => [''.upcase],
+          'mpc_sample_design' => [''.upcase],
+          'sample_unit' => ['microdata sample characteristics:sample unit'.upcase],
+          'sample_fraction' => ['microdata sample characteristics:sample fraction'.upcase],
+          'sample_fraction_notes' => [''.upcase],
+          'sample_size' => ['microdata sample characteristics:sample size (person records)'.upcase],
+          'sample_weights' => ['microdata sample characteristics:sample weights'.upcase],
+          'sample_characteristics_notes' => [''.upcase],
+          'sample_general_notes' => [''.upcase],
+
+          'has_dwellings' => ['units identified:dwellings'.upcase],
+          'has_dwellings_note' => [''.upcase],
+          'has_vacant_units' => ['Units identified:Vacant units'.upcase, 'Units identified:Vacant dwellings'.upcase],
+          'has_vacant_units_note' => [''.upcase],
+          'has_closed_units' => [''.upcase],
+          'has_closed_units_note' => [''.upcase],
+          'smallest_geography' => ['units identified:smallest geography'.upcase],
+
+          'has_households' => ['units identified:households'.upcase],
+          'has_households_note' => [''.upcase],
+          'has_families' => ['units identified:families'.upcase],
+          'has_families_note' => [''.upcase],
+          'has_individuals' => ['units identified:individuals'.upcase],
+          'has_individuals_note' => [''.upcase],
+          'has_group_quarters' => ['Units identified:Group quarters'.upcase, 'Units identified:Group quarters (Collective households)'.upcase],
+          'has_group_quarters_note' => [''.upcase],
+          'has_indigenous_pop' => ['units identified:indigenous population'.upcase],
+          'has_indigenous_pop_note' => [''.upcase],
+          'has_special_pop' => ['units identified:special populations'.upcase],
+          'has_special_pop_note' => [''.upcase],
+          'units_notes' => [''.upcase],
+          'identification_general_notes' => [''.upcase],
+
+          'unit_definition_household' => ['Unit definitions:Households'.upcase, 'Unit definitions:Private Household'.upcase],
+          'unit_definition_family' => ['unit definitions:families'.upcase],
+          'unit_definition_group_quarters' => ['Unit definitions:Group quarters'.upcase, 'Unit definitions:Group Quarters (Collective households)'.upcase],
+          'unit_definition_dwelling' => ['unit definitions:dwellings'.upcase],
+          'unit_definition_homeless_population' => [''.upcase],
+          'unit_definition_institution' => [''.upcase],
+          'unit_definition_institutional_population' => [''.upcase],
+          'unit_definition_notes' => [''.upcase],
+          'unit_definition_general_notes' => [''.upcase]
+      }
+
+      @country_abbrev_lookup = Hash[Country.where({}).order(:short_name).map{|country| [country.short_name, country.full_name]}]
+    
+      @@has_startup = true
+    end
+    
+  end
+  
+  def self.parse(string)
+    if string =~ /\A[Yy]es\Z/
+      # is it a 'yes'?
+      true
+    elsif string =~ /\A[Nn]o\Z/ or string =~ /Not available in microdata sample/ or string =~ /---/
+      #is it a 'no'?
+      false
+    elsif string =~ /\A\d+(.\d+)?%\Z/
+      # is it a percentage? (which could look like 100% or like 10.5%)
+      numeric_portion = string.chop
+      (numeric_portion.to_i) / 100.0
+    elsif string =~ /\A\d+\Z/ or string =~ /\A(\d{1,3},?)+\Z/
+      # is it an int with or without commas?
+      numeric_portion = string.gsub(',', '')
+      numeric_portion.to_i
+    else
+      # just return the argument itself.
+      string
+    end
+  end
+
+  # country code is in the form <2-char country id><4-digit year><sample_id>
+  # such as us1990a for the US 1990 census.
+  # returns a hash with keys of :abbrev, :country and :year
+  # or nil if the name doesn't match the regex.
+  def self.parse_sample_id(name)
+    # $stderr.puts name
+    code_regex = /(?<abbrev>[a-z]{2})(?<year>\d{4})(?<sample_id>[a-z])/
+    match = code_regex.match(name)
+    if match.nil?
+      nil
+    else
+      # $stderr.puts "==> #{match[:abbrev]}"
+      {:abbrev => match[:abbrev], :year => match[:year], :country => @country_abbrev_lookup[match[:abbrev]]}
+    end
+  end
+
+  # returns the dataset entry as a hash ready to be converted to YAML.
+  def self.construct_sample_entry(possible_tags, name, contents)
+    sampledesc_hash = {'sample_id' => name}
+    output = {}
+    possible_tags.each { |key, value|
+      if value.count > 0
+        sampledesc_key = value.detect { |sampledesc_key| contents.has_key? (sampledesc_key) }
+        unless sampledesc_key.nil?
+          unless contents[sampledesc_key].nil?
+            sampledesc_hash[key] = parse(contents[sampledesc_key].strip)
+          end
+        end
+      end
+
+      output = {'terrapop_samples' => [sampledesc_hash]}
+    }
+    output
+  end
+
+  # Return an array of sample_geog_level instances.
+  # For now, just construct national and GEOLEV1 entries here. As more are added, add more.
+  #
+  def self.construct_sample_geog_levels(sample_id, abbrev, name, year, contents)
+    national = {
+        'terrapop_sample_id' => "#{name} #{year}",
+        'label' => "#{name}: National, #{year}",
+        'internal_code' => "#{sample_id}_NAT",
+        'geolink_variable_id' => 'CNTRY',
+        'country_level_id' => [{'geog_unit_id' => 'NAT'}, {'country_id' => "#{abbrev}"}]
+    }
+    geolev1 = {
+        'terrapop_sample_id' => "#{name} #{year}",
+        'label' => "#{name}: First Level (Harmonized), #{year}",
+        'internal_code' => "#{sample_id}_HFLAD",
+        'geolink_variable_id' => 'GEOLEV1',
+        'country_level_id' => [{'geog_unit_id' => 'HFLAD'}, {'country_id' => "#{abbrev}"}]
+    }
+    [national, geolev1]
+
+  end
+  
+  def self.to_geog_setup(country)
+    
+    unless country.kind_of? Country
+      raise "Object passed to SampleDesign::to_geog_setup must be of type Country"
+    end
+    
+    sd = self.where({:country_id => country.id}).first
+    
+    if sd.nil?
+      $stderr.puts "==> country_id #{country.id} [#{country.full_name}] not found for SampleDesign"
+    else
+
+      startup
+
+      column = -1
+      cat    = key = ''
+      result = {}
+    
+      contents = nil
+    
+      begin
+        contents = CSV.parse(sd.document, {headers: true, col_sep: "\t", encoding: "iso-8859-1:utf-8"}) # CSV.read(path, {headers: true, col_sep: "\t", encoding: "iso-8859-1:utf-8"})
+      rescue Exception => e
+        throw e
+      end
+    
+      unless contents.nil?
+        # now flip it over.
+
+        # first, drop the first two elements from the header to get the dataset names.
+        dataset_titles = contents.headers
+        dataset_titles.shift
+        dataset_titles.shift
+
+        # make a hash of hashes to hold the flipped table
+        datasets = {}
+        # and set up the sample label for each entry, initialized with the label field.
+        dataset_titles.each { |ds|
+          label_info = parse_sample_id(ds)
+          unless label_info.nil?
+            datasets[ds] = {'CUSTOM:LABEL' => "#{label_info[:country]} #{label_info[:year]}"}
+          end
+        }
+
+        # now run through the rows of the CSV and build out the dataset sample descriptions
+        category = 'heading'
+        contents.each { |row|
+          
+          unless row['item'].nil?
+            # puts row.inspect
+            this_field = row['item'].strip
+            this_category = row['heading']
+            category = this_category.strip unless this_category.nil? or this_category.empty?
+
+            field = "#{category}:#{this_field}".upcase # get the field heading
+
+            # and then get the value of that field heading for each dataset,
+            # and put it into the appropriate subhash in the datasets hash.
+            dataset_titles.each { |ds|
+              if datasets.has_key? ds
+                datasets[ds][field] = row[ds]
+              end
+            }
+          end
+          
+        }
+
+        datasets.each { |name, contents|
+          $stderr.puts " +++ Working on dataset[#{name}]...".color(:green)
+          File.open(File.join(Rails.root.to_s, "data/geog_setup", "#{name}_geog_setup.yml"), 'w+') {|file|
+            $stderr.puts " +++ Writing #{file.path}..."
+            sample_info = parse_sample_id(name)
+            terrapop_sample = construct_sample_entry(@tag_mapping, name, contents)
+            terrapop_sample['terrapop_samples'][0]['short_country_name'] = country.short_name
+            terrapop_sample['terrapop_samples'][0]['year'] = terrapop_sample['terrapop_samples'][0]['sample_id'][2,4].to_i
+            # $stderr.puts terrapop_sample.to_s
+            geog_info = construct_sample_geog_levels(name, sample_info[:abbrev], sample_info[:country],sample_info[:year], contents)
+            terrapop_sample['sample_geog_levels'] = geog_info
+            file.puts terrapop_sample.to_yaml
+          }
+        }
+    
+        {status: :OK}
+      else
+        {status: :FAILED, message: "#{country.full_name} failed to produce geog_setup yaml..."}
+      end
+    end
+  end
+  
+  def self.fields(sample)
+    
+    unless sample.kind_of? Sample
+      raise "Object passed to SampleDesign::fields must be of type Sample"
+    end
+    
+    sd = self.where({:country_id => sample.country_id}).first
+    
+    if sd.nil?
+      raise "country_id #{sample.country_id} not found for SampleDesign"
+    end
+
+    column = -1
+    cat    = key = ''
+    result = {}
+
+    CSV.parse(sd.document, :col_sep => "\t", :row_sep => "\r\n").each_with_index do |record, record_no|
+      record.each_with_index do |field, field_no|
+        next if field.nil?
+        field.strip!
+        case record_no
+          when 0 then column = field_no if field == sample.name # find the sample's column in the first line
+          else # all other lines, build the result hashtable
+            case field_no
+              when 0 then (cat = field) unless field.empty?
+              when 1 then (key = cat + ':' + field) unless field.empty?
+              when column then result[key] = field
+            end
+        end
+      end
+    end
+
+    result
+  end
+
+  
+  
+end

data/app/models/sample_detail_field.rb

@@ -0,0 +1,7 @@
+class SampleDetailField < ActiveRecord::Base
+  belongs_to :sample_detail_group
+
+  validates :sample_detail_group, presence: true
+  validates :name, presence: true
+  validates :label, presence: true
+end

data/app/models/sample_detail_group.rb

@@ -0,0 +1,3 @@
+class SampleDetailGroup < ActiveRecord::Base
+  validates :name, presence: true
+end

data/app/models/sample_detail_value.rb

@@ -0,0 +1,11 @@
+class SampleDetailValue < ActiveRecord::Base
+  belongs_to :sample_detail_field
+  belongs_to :sample
+
+  validates :sample_detail_field, presence: true
+  validates :sample, presence: true
+
+  def to_s
+    self.value
+  end
+end

data/app/models/sample_details.rb

@@ -0,0 +1,90 @@
+class SampleDetails
+
+  class Collection
+    def initialize(samples, groups, fields, values)
+      @samples = samples.to_a
+      @countries = @samples.map { |s| s.country }.uniq.sort { |a, b| a.full_name <=> b.full_name }
+      @groups = groups.to_a.sort { |a, b| a.order <=> b.order }
+      @fields = fields.to_a
+      @values = values.to_a
+
+      sample_map = Hash[@samples.map { |s| [s.id, s] }]
+      @value_sample_map = {}
+      @group_sample_map = {}
+
+      @values.each do |v|
+        s = sample_map[v.sample_id]
+        g = v.sample_detail_field.sample_detail_group
+        (@value_sample_map[s] ||= []) << v
+        (@group_sample_map[s] ||= {})[g] = nil
+      end
+
+      @group_sample_map.each { |s, v| @group_sample_map[s] = v.keys }
+    end
+
+    def countries
+      @countries
+    end
+
+    def samples(for_country = nil)
+      if for_country
+        @samples.select { |s| s.country == for_country }
+      else
+        @samples
+      end
+    end
+
+    def groups(used_by_sample = nil)
+      if used_by_sample
+        @group_sample_map[used_by_sample] || []
+      else
+        @groups
+      end
+    end
+
+    def fields(for_group = nil, exclude_summary_only = false)
+      if for_group
+        data = @fields.select { |f| f.sample_detail_group == for_group }
+      else
+        data = @fields
+      end
+
+      if exclude_summary_only
+        data.reject { |f| f.summary_only }
+      else
+        data
+      end
+    end
+
+    def value(sample, field)
+      field = case field
+                when SampleDetailField
+                  field
+                when String
+                  @fields.detect { |f| f.name == field }
+                else
+                  nil
+              end
+
+      return nil unless sample && field
+
+      list = @value_sample_map[sample] || []
+      list.detect { |v| v.sample_detail_field_id == field.id }
+    end
+
+  end
+
+  # Returns a SampleDetails::Collection object for the given samples (which should be a relation or list of AR Sample objects)
+  # optionally, if detail_fields are specified, only those details fields will be returned (expects a list of strings)
+  def self.for(samples, detail_fields = [])
+    fields = SampleDetailField.eager_load(:sample_detail_group).order(:order)
+    values = SampleDetailValue.preload({:sample_detail_field => :sample_detail_group}).where(sample_id: samples)
+
+    if detail_fields && !detail_fields.empty?
+      fields = fields.where(name: detail_fields)
+      values = values.where(sample_detail_field_id: fields)
+    end
+
+    Collection.new(samples, fields.map { |f| f.sample_detail_group }.uniq, fields, values)
+  end
+end

data/app/models/sample_geog_level.rb

@@ -0,0 +1,150 @@
+class SampleGeogLevel < ActiveRecord::Base
+
+  has_many :geog_instances
+  has_many :sample_level_area_data_variables
+  has_many :request_raster_variables
+  has_many :request_area_data_variables
+  has_many :area_data_rasters
+  has_many :area_data_variables, through: :sample_level_area_data_variables
+
+  belongs_to :country_level
+  belongs_to :terrapop_sample
+  # geolinking_variable is the id of the variable to link with when attaching area-level characteristics to microdata extracts.
+  belongs_to :geolink_variable, class_name: 'Variable', foreign_key: 'geolink_variable_id'
+
+
+  def api_attributes
+    "TODO"
+  end
+
+
+  def geog_unit
+    country_level.geog_unit
+  end
+
+
+  def map
+    Map.where(country_level_id: country_level_id, terrapop_sample_id: terrapop_sample_id).first
+  end
+
+
+  def self.long_description(sample_geog_levels)
+    if sample_geog_levels.count > 0
+      level_order = ['NAT', 'HFLAD', 'FLAD', 'HSLAD', 'SLAD']
+      levels = {}
+      str = []
+      sample_geog_levels.each do |sgl|
+        levels[sgl.country_level.geog_unit.code] = [] unless levels.has_key?(sgl.country_level.geog_unit.code)
+        levels[sgl.country_level.geog_unit.code] << sgl.long_description
+      end
+      levels.each{ |key, lvls| levels[key] = lvls.sort }
+      level_order.each do |lvl|
+        if levels.has_key? lvl
+          str << GeogUnit.where(code: lvl).first.label + " (" + lvl + ")"
+          levels[lvl].each{ |lvl_txt| str << "  " + lvl_txt }
+        end
+      end
+      str.join("\n")
+    else
+      "No Geographic Levels"
+    end
+  end
+
+
+  def long_description
+    terrapop_sample.country_name_long_year + ": " + (country_level.geog_unit.code == "NAT" ? "National" : country_level.label)
+  end
+
+  # Use the geolinking variable from the microdata to help generate the
+  # AreaVariable stubs that are the geography instances
+  # on each row of the area data tables. We need four:
+  # 1. geog level label (State, county, etc.)
+  # 2. geog level code: (STATEBR, STATEUS, etc.)
+  # 3. geog instance label ("Alabama","Hennepin", etc.)
+  # 4. geog instance code (25,28, whatever codes are used for categories of 3
+  def variables_for_area_data_extracts
+    geog_instance_variable = "GEOG_CODE"
+    geog_instance_variable = geolink_variable.mnemonic || "GEOG_CODE" unless geolink_variable.nil?
+    geog_instance_label = "#{geog_instance_variable}_LABEL"
+    
+    adrvml_code = AreaDataRasterVariableMnemonicLookup.where(composite_mnemonic: geog_instance_variable).first
+    
+    if adrvml_code.nil?
+      adrvml_code = AreaDataRasterVariableMnemonicLookup.new
+      adrvml_code.description = 'GIS match code'
+      
+      unless geolink_variable.nil?
+        adrvml_code.description += ' [' + geolink_variable.label + ']'
+      end
+      
+      adrvml_code.composite_mnemonic = geog_instance_variable
+      adrvml_code.mnemonic = geog_instance_variable
+
+      adrvml_code.save
+    end
+    
+    adrvml_label = AreaDataRasterVariableMnemonicLookup.where(composite_mnemonic: geog_instance_label).first
+    
+    if adrvml_label.nil?
+      adrvml_label = AreaDataRasterVariableMnemonicLookup.new
+      adrvml_label.description = 'Name of geographic instances'
+      adrvml_label.composite_mnemonic = geog_instance_label
+      adrvml_label.mnemonic = geog_instance_label
+      adrvml_label.save
+    end
+    
+    [
+      # ExtractVariableStub.new(:mnemonic => "Geog_level_label",     :label=>"Geog level label",    :len => 32, :data_type => "alphabetical"),
+      # ExtractVariableStub.new(:mnemonic => "Geog_level_code",      :label=>"Geog level code",     :len => 10, :data_type => "alphabetical"),
+      ExtractVariableStub.new(mnemonic: geog_instance_label,    label: "Geog instance label", len: 32, data_type: "alphabetical"),
+      ExtractVariableStub.new(mnemonic: geog_instance_variable, label: "Geog instance code",  len: 10, data_type: "integer")
+    ]
+  end
+
+  #from_nhgis_terrapop_sample presumes that the terrapop sample is associated to an NHGIS dataset; if it is not, return an empty set
+  # the number of sample geog levels returned depends upon the number of country levels associated to the terrapop
+  # sample; e.g. 3: one for US-NAT (United States), one for US-HFLAD (states), and one for US-HSLAD (counties)
+  def self.from_nhgis_terrapop_sample(nhgis_terrapop_sample)
+    #return an empty list if the terrapop sample is not linked to an NHGIS dataset
+    return [] if nhgis_terrapop_sample.nhgis_dataset_id.nil?
+    #TODO create a relation in Terrapop Sample?  too drastic
+    nhgis_dataset = Nhgis::Dataset.where(id: nhgis_terrapop_sample.nhgis_dataset_id).first
+    return [] if nhgis_dataset.nil?
+    #identify the country for the TerrapopSample
+    country = nhgis_terrapop_sample.country
+    raise "expecting the terrapop sample label to include the name of the country '#{country.full_name}'" unless nhgis_terrapop_sample.label.include? country.full_name
+    # only those NHGIS datasets having nation-, state-, or county-level data are ever to be considered for terrapop samples.  There are some NHGIS datasets that
+    # participate in time series tables but do not have any nation-, state-, or county-level data.  Since Terrapop only has NAT, *FLAD, *SLAD, those datasets are
+    # to be excluded.
+    # conversion from NHGIS geog levels/geog units to Terrapop geog units
+    nhgis_gl_to_terrapop_gu = {"nation" => "NAT", "state" => "FLAD", "county" => "SLAD"}
+    valid_geog_units = nhgis_dataset.data_groups.reject{ |dg| dg.relative_pathname.nil? }.map{ |dg| dg.relative_pathname.nil? ? nil : nhgis_gl_to_terrapop_gu[dg.geog_level.istads_id] }.reject{ |gl_istads_id| gl_istads_id.nil? }
+    geog_unit_ids = GeogUnit.where(code: valid_geog_units).pluck(:id)
+    #identify all of the country levels for the terrapop sample's country
+    country_levels = CountryLevel.where(country_id: nhgis_terrapop_sample.country_id, geog_unit_id: geog_unit_ids).all
+    country_levels.map do |country_level|
+      sgl = SampleGeogLevel.new
+      sgl.country_level = country_level
+      sgl.terrapop_sample = nhgis_terrapop_sample
+      gu = country_level.geog_unit
+      sgl.internal_code = "#{country.short_name}#{nhgis_dataset.code}_#{gu.code}"
+      harmonized = ["HFLAD", "HSLAD"].include?(gu.code) ? " (Harmonized)" : "" #National country level need not be harmonized; only states and counties are harmonized
+      country_level_label = gu.code == "NAT" ? gu.label : country_level.label #use the geog unit label... which is expected to be 'National' (whereas United States represents extent)
+      sgl.label = nhgis_terrapop_sample.label.gsub(country.full_name, "#{country.full_name}: #{country_level_label}#{harmonized},")
+      sgl #do not set values for id, created_at, updated_at, code, and geolink_variable
+    end
+  end
+
+  def self.geography_name(sample_geog_level)
+    if sample_geog_level.nil?
+      ""
+    else
+      if sample_geog_level.geolink_variable.nil?
+        sample_geog_level.terrapop_sample.short_country_name.upcase + "_" + sample_geog_level.country_level.geog_unit.code.upcase
+      else
+        sample_geog_level.geolink_variable.mnemonic
+      end
+    end
+  end
+
+end