terrapop_models 0.1.0

data/app/models/extract_request_area_data_raster_variable_mnemonic_lookup.rb

@@ -0,0 +1,4 @@
+class ExtractRequestAreaDataRasterVariableMnemonicLookup < ActiveRecord::Base
+  belongs_to :extract_request
+  belongs_to :area_data_raster_variable_mnemonic_lookup
+end

data/app/models/extract_request_error_event.rb

@@ -0,0 +1,4 @@
+class ExtractRequestErrorEvent < ActiveRecord::Base
+  belongs_to :error_event
+  belongs_to :extract_request
+end

data/app/models/extract_request_submission.rb

@@ -0,0 +1,19 @@
+class ExtractRequestSubmission < ActiveRecord::Base
+  belongs_to :extract_request
+  
+  scope :most_recent, -> { joins("INNER JOIN extract_requests ON extract_requests.id = extract_request_submissions.extract_request_id").where("extract_requests.id = extract_request_submissions.extract_request_id").order("extract_request_submissions.submitted_at DESC") }
+  
+  def self.dump_all
+    {
+      self.to_s.underscore.pluralize => self.all.map{|ers|
+        unless ers.extract_request.user_id.nil?
+          {
+            'extract_request_id' => ers.extract_request.uuid,
+            'submitted_at'       => ers.submitted_at
+          }
+        end
+      }.reject{|x| x.nil? }
+    }
+  end
+  
+end

data/app/models/extract_status.rb

@@ -0,0 +1,55 @@
+require 'status_definition'
+
+class ExtractStatus < ActiveRecord::Base
+
+  # see data/static_data.yml for the available status_definitions
+  belongs_to :extract_request
+  scope :extracts, -> {includes(:extract_request)}
+
+  scope :current, -> {where("(extract_statuses.id, extract_statuses.extract_request_id) in (select max(extract_statuses.id) as max_id, extract_request_id from extract_statuses group by extract_request_id)")}
+
+  def self.define_consts
+
+    @@status_definitions = []
+
+    @@status_definitions = status_definitions = StatusDefinition.where({})
+
+    status_definitions.each { |status_definition|
+      status = status_definition.status
+
+      unless status.nil?
+        status = status.downcase
+        self.const_set status.upcase, status
+        scope status, -> {where(:status => status)}
+      else
+        raise "Your database maybe in an unknown state; ExtractStatus' status is nil"
+      end
+    }
+
+  end
+
+  def self.const_missing(name)
+    name.downcase.to_s
+  end
+
+  def self.status_definitions
+    @@status_definitions
+  end
+
+  def self.dump_all
+    {
+      self.to_s.underscore.pluralize => self.where(["status <> 'building_request'"]).map{|es|
+        unless es.extract_request.user_id.nil?
+          {
+            'extract_request_id' => es.extract_request.uuid,
+            'status'             => es.status
+          }
+        end
+      }.reject{|x| x.nil? }
+    }
+  end
+
+
+end
+
+ExtractStatus.define_consts

data/app/models/extract_type.rb

@@ -0,0 +1,2 @@
+class ExtractType < ActiveRecord::Base
+end

data/app/models/extract_variable_stub.rb

@@ -0,0 +1,17 @@
+# For area-level extracts, we include geog_level and geog_instance fields,
+# but these aren't part of the request structure, which means that they need to be provided
+# to the syntax generation system directly. This stub class provides a way to do that.
+# data_type is expected to map to "integer" or "alphabetical"
+class ExtractVariableStub
+  attr_reader :mnemonic,:len,:label,:implied_decimal_places,:data_type,:categories
+          
+  def initialize(params)
+    @mnemonic = params[:mnemonic] or raise "Must set mnemonic."
+    @len = params[:len] or raise "Must set len."
+    @label = params[:label] or raise "Must set label."
+    @data_type = params[:data_type] or raise "Must set data_type."
+    @categories = params[:categories] || []
+    @implied_decimal_places = params[:implied_decimal_places] || 0
+  end
+
+end

data/app/models/frequency.rb

@@ -0,0 +1,7 @@
+class Frequency < ActiveRecord::Base
+
+  belongs_to :category
+  belongs_to :sample
+  belongs_to :variable
+
+end

data/app/models/geog_instance.rb

@@ -0,0 +1,18 @@
+class GeogInstance < ActiveRecord::Base
+
+  has_many :boundaries
+  has_many :raster_categories
+  has_many :raster_statistics
+  has_many :raster_category_statistics
+  has_many :area_data_statistics
+
+  belongs_to :sample_geog_level
+  belongs_to :terrapop_sample
+
+  belongs_to :parent, :class_name => "GeogInstance", :foreign_key => "parent_id"
+  has_many :children, :class_name => "GeogInstance", :foreign_key => "parent_id"
+
+  def to_s
+    label.nil? ? "<not set>" : label + "(#{code})"
+  end
+end

data/app/models/geog_unit.rb

@@ -0,0 +1,5 @@
+class GeogUnit < ActiveRecord::Base
+
+  has_many :country_levels
+
+end

data/app/models/global_region.rb

@@ -0,0 +1,6 @@
+class GlobalRegion < ActiveRecord::Base
+
+  has_many   :countries
+  belongs_to :global_region_type
+
+end

data/app/models/global_region_type.rb

@@ -0,0 +1,5 @@
+class GlobalRegionType < ActiveRecord::Base
+
+  has_many :global_region
+
+end

data/app/models/heartbeat.rb

@@ -0,0 +1,8 @@
+class Heartbeat < ActiveRecord::Base
+  has_many :heartbeat_pulses
+
+  def cart_time
+    return self.heartbeat.heartbeat_pulses.count * 3
+  end
+
+end

data/app/models/heartbeat_pulse.rb

@@ -0,0 +1,3 @@
+class HeartbeatPulse < ActiveRecord::Base
+  belongs_to :heartbeat
+end

data/app/models/insert_html_fragment.rb

@@ -0,0 +1,3 @@
+class InsertHtmlFragment<ActiveRecord::Base
+  
+end

data/app/models/ipums_academic_status.rb

@@ -0,0 +1,9 @@
+require 'ipumsi_database'
+
+class IpumsAcademicStatus < IpumsiActiveRecord::Base
+  self.table_name = :academic_statuses
+  self.primary_key = :id
+
+  scope :for_web, -> { where.not(web_text: nil).order(:web_order) }
+
+end

data/app/models/ipums_anticipated_result.rb

@@ -0,0 +1,9 @@
+require 'ipumsi_database'
+
+class IpumsAnticipatedResult < IpumsiActiveRecord::Base
+  self.table_name = :anticipated_results
+  self.primary_key = :id
+
+  scope :for_web, -> { where.not(web_text: nil).order(:web_order) }
+
+end

data/app/models/ipums_department.rb

@@ -0,0 +1,9 @@
+require 'ipumsi_database'
+
+class IpumsDepartment < IpumsiActiveRecord::Base
+  self.table_name = :departments
+  self.primary_key = :id
+
+  scope :for_web, -> { where.not(web_text: nil).order(:web_order) }
+
+end

data/app/models/ipums_registration.rb

@@ -0,0 +1,72 @@
+require 'ipumsi_database'
+
+class IpumsRegistration < IpumsiActiveRecord::Base
+  self.table_name = :registrations
+  self.primary_key = :id
+
+  attr_accessor :allow_emails, :ownership, :use, :restriction, :confidentiality, :security, :appropriate_citation, :violations, :notify_errors
+
+  validates :id, presence: true, uniqueness: true
+  validates :address_line_1, :city, :state, :zip_code, :country, :country_of_origin, :phone, :approval_status, :expires_at, :last_renewed_at, presence: true
+  validates :allow_emails, :ownership, :use, :restriction, :confidentiality, :security, :appropriate_citation, :violations, :notify_errors, acceptance: true
+  validates :institutional_affiliation, :health_research, inclusion: {in: [true, false]}
+  validates :department_id, inclusion: {in: IpumsDepartment.for_web.ids }
+  validates :department_text, presence: true, if: :department_is_other?
+  validates :academic_status_id, inclusion: {in: IpumsAcademicStatus.for_web.ids }
+  validates :academic_status_text, presence: true, if: :academic_status_is_other?
+  validates :anticipated_result_id, inclusion: {in: IpumsAnticipatedResult.for_web.ids }
+  validates :anticipated_result_text, presence: true, if: :anticipated_result_is_other?
+  validates :why_no_institution, presence: true, unless: :institutional_affiliation?
+  validates :institution_name, presence: true, if: :institutional_affiliation?
+  validates :has_safety_board, inclusion: {in: [true, false], message: "must be selected"}, if: :institutional_affiliation?
+  validates :safety_board_name, presence: {message: "can't be blank if your institution have an Institutional Review Board (IRB), or Office for Human Subject Protections, Professional Conduct or similar committee"}, if: [:institutional_affiliation?, :has_safety_board?]
+  validates :research_description, length: {minimum: 75, tokenizer: lambda { |str| str.split(/\s+/) }, too_short: "Research Description must have at least %{count} words"}
+
+  STATUS = {approved: "approved", denied: "denied", pending: "pending", incomplete: "incomplete"}
+
+
+  def set_default_creation_attributes
+    set_as_renewed
+    self.approval_status  = Rails.env.live? ? STATUS[:pending] : STATUS[:approved]
+    self.max_extract_size = 20480
+  end
+
+
+  def set_as_renewed
+    now = DateTime.now
+    self.expires_at      = now + 1.year - 1.day
+    self.last_renewed_at = now
+  end
+
+
+  def is_expired?
+    expires_at.nil? || DateTime.now > expires_at
+  end
+
+
+  private
+
+
+
+  def department_is_other?
+    department_id.blank? and return false
+    department = IpumsDepartment.find_by(id: department_id)
+    !department.nil? && ['academic_other', 'nonacademic_other'].include?(department.label)
+  end
+
+
+  def academic_status_is_other?
+    academic_status_id.blank? and return false
+    academic_status = IpumsAcademicStatus.find_by(id: academic_status_id)
+    !academic_status.nil? && ['other_academic_other', 'nonacademic'].include?(academic_status.label)
+  end
+
+
+  def anticipated_result_is_other?
+    anticipated_result_id.blank? and return false
+    anticipated_result = IpumsAnticipatedResult.find_by(id: anticipated_result_id)
+    !anticipated_result.nil? && anticipated_result.label == 'other'
+  end
+
+
+end

data/app/models/ipums_user.rb

@@ -0,0 +1,15 @@
+require 'ipumsi_user_database'
+
+class IpumsUser < IpumsiUserActiveRecord::Base
+  self.table_name = :users
+
+  validates :first_name, :last_name, :salt, :crypted_password, presence: true
+  validates :email, uniqueness: true, format: {with: /\A[^@]+@[^@]+\.[^@]+\z/}
+
+
+  def get_ipumsi_registration
+    IpumsRegistration.find_by(id: self.id)
+  end
+
+
+end

data/app/models/label.rb

@@ -0,0 +1,7 @@
+class Label < ActiveRecord::Base
+
+  self.table_name = "tags"
+
+  has_and_belongs_to_many :extract_requests, :join_table => "extract_requests_labels"
+
+end

data/app/models/link.rb

@@ -0,0 +1,21 @@
+class Link < ActiveRecord::Base
+
+  def self.hash
+    @@memo ||= make_hash
+  end
+
+
+  def self.make_hash
+    hash = {}
+    self.all.each do |link|
+      hash[link.name] = link.location
+    end
+    hash
+  end
+
+
+  def self.clear_cached
+    @@memo = nil
+  end
+
+end

data/app/models/map.rb

@@ -0,0 +1,255 @@
+class Map < ActiveRecord::Base
+
+  has_many :boundaries
+  belongs_to :country
+  belongs_to :country_level
+  belongs_to :terrapop_sample
+
+  alias_attribute :dataset_id, :terrapop_sample_id
+
+
+  # Be sure to call construct_instances before calling construct_boundaries
+  def construct_instances_for_terrapop_sample(tps, parent_level = nil, code_column = 'ipums_code', include_numeric_code = true)
+    my_sgl = tps.sample_geog_level_for_country_level(self.country_level)
+    unless my_sgl.nil?
+      geog_unit_code = my_sgl.country_level.geog_unit.code
+
+      if self.terrapop_sample_id.nil?
+        self.terrapop_sample_id = tps.id
+        self.save
+      end
+
+      geog_code = nil
+      short_name = tps.country.short_name.upcase #my_sgl.terrapop_sample.country.short_name.upcase
+
+      if geog_unit_code == 'NAT'
+        geog_code = 'CNTRY'
+      elsif geog_unit_code == 'HFLAD'
+        geog_code = 'GEOLEV1'
+      elsif geog_unit_code == 'FLAD'
+        geog_code = 'GEO1B_' + short_name
+      elsif geog_unit_code == 'HSLAD'
+        geog_code = 'GEO2A_' + short_name
+      elsif geog_unit_code == 'SLAD'
+        geog_code = 'GEO2B_' + short_name
+      end
+
+      #$stderr.puts "----------------------------------------------------------------------------"
+      #$stderr.puts "#{my_sgl.terrapop_sample.country.full_name} => #{my_sgl.country_level.geog_unit.code} ==> #{geog_code.to_s}"
+      #$stderr.puts "----------------------------------------------------------------------------"
+
+      # the insert statement for the national level is simpler than others, so if parent-level is nil.
+      # check that we're doing national.
+      #if parent_level.nil? and my_sgl.internal_code.rindex('NAT').nil?
+      #  raise "attempt to call construct_instances_for_terrapop_sample with no parent_level: #{my_sgl.internal_code}"
+      #end
+
+      # once there's a shape_area column on all the maps, we can populate that from here
+
+      inc = include_numeric_code
+
+      geog_instances_count = GeogInstance.where({sample_geog_level_id: my_sgl.id, geog_code: geog_code.to_s}).count
+
+      #GeogInstance.delete_all(["sample_geog_level_id = ? AND geog_code = ?", my_sgl.id, geog_code.to_s])
+
+      if geog_instances_count == 0
+
+        insert_stmt = if parent_level.nil?
+          "insert into geog_instances(sample_geog_level_id, parent_id, #{inc ? 'code,' : ''} label, geog_code, str_code, terrapop_sample_id)
+          select #{my_sgl.id}, NULL, #{inc ? "CAST(shape.#{code_column} as numeric) as code," : ''} shape.label, '#{geog_code.to_s}', shape.#{code_column}, #{tps.id}
+          from #{self.shape_table_name} shape WHERE shape.#{code_column} IS NOT NULL"
+        else
+          "insert into geog_instances(sample_geog_level_id, parent_id, #{inc ? 'code, ' : ''} label, geog_code, str_code, terrapop_sample_id)
+            select #{my_sgl.id}, parent.id, #{inc ? "CAST(shape.#{code_column} as numeric) as code," : ''} shape.label, '#{geog_code.to_s}', shape.#{code_column}, #{tps.id}
+            from #{self.shape_table_name} shape
+            inner join geog_instances parent ON parent.code = CAST(shape.parent as numeric) AND parent.sample_geog_level_id = #{parent_level.id} WHERE shape.#{code_column} IS NOT NULL"
+            # inner join geog_instances parent ON parent.code = CAST(shape.parent as numeric) and parent.sample_geog_level_id = #{parent_level.id} WHERE shape.#{code_column} IS NOT NULL
+        end
+
+        records = ActiveRecord::Base.connection.execute(insert_stmt)
+
+      else
+        $stderr.puts "Found GeogInstances for SampleGeogLevel[#{my_sgl.id}], geog_code: #{geog_code.to_s}"
+
+        #if parent_level.nil?
+          sql = "select #{my_sgl.id} AS sample_geog_level_id, NULL AS parent_id, #{inc ? "CAST(shape.#{code_column} as numeric) as code," : ''} shape.label AS label, '#{geog_code.to_s}' AS geog_code, shape.#{code_column} AS str_code, #{tps.id} AS terrapop_sample_id FROM #{self.shape_table_name} shape WHERE shape.#{code_column} IS NOT NULL"
+        #else
+        #  sql = "select #{my_sgl.id} AS sample_geog_level_id, parent.id AS parent_id, #{inc ? "CAST(shape.#{code_column} as numeric) as code," : ''} shape.label AS label, '#{geog_code.to_s}' AS geog_code, shape.#{code_column} AS str_code, #{tps.id} AS terrapop_sample_id from #{self.shape_table_name} shape inner join geog_instances parent ON parent.code = CAST(shape.parent as numeric) and parent.sample_geog_level_id = #{parent_level.id} WHERE shape.#{code_column} IS NOT NULL"
+        #end
+
+        records = ActiveRecord::Base.connection.execute(sql)
+
+        geog_instances = GeogInstance.where({sample_geog_level_id: my_sgl.id, geog_code: geog_code.to_s})
+
+        geog_instances.each{|geog_instance|
+
+          records.each{|record|
+
+            if geog_instance.label == record['label']
+              geog_instance.parent_id = record['parent_id']
+
+              if inc
+                geog_instance.code = record['code']
+              end
+
+              geog_instance.geog_code = record['geog_code']
+              geog_instance.str_code  = record['str_code']
+
+              geog_instance.save
+
+            end
+
+          }
+        }
+
+      end
+    else
+      $stderr.puts "=*=*=> SampleGeogLevel for #{tps.inspect} at #{self.country_level.inspect}"
+    end
+  end
+
+  def construct_boundaries_for_terrapop_sample(tps, code_column = 'ipums_code')
+    my_sgl = tps.sample_geog_level_for_country_level(self.country_level)
+    unless my_sgl.nil?
+
+      boundaries_count = Boundary.where({map_id: self.id}).count
+
+      if boundaries_count > 0
+        $stderr.puts " Found Boundary[map_id: #{self.id}]"
+        #Boundary.where({map_id: self.id}).destroy_all
+      else
+        insert_stmt =
+          "insert into boundaries(map_id, geog_instance_id, code, description, geog, geom)
+            select #{self.id}, gi.id, CAST(shape.#{code_column} as numeric) as code, shape.label, shape.geog, shape.geom
+            from #{self.shape_table_name} shape
+            inner join geog_instances gi
+              on gi.sample_geog_level_id = #{my_sgl.id} and gi.code = CAST(shape.#{code_column} as numeric) WHERE shape.#{code_column} <> ''"
+
+        records = ActiveRecord::Base.connection.insert(insert_stmt)
+      end
+    end
+  end
+
+  def construct_instances_for_nhgis_terrapop_sample(nhgis_tps, sgl, parent_sgl = nil)
+    return nil if nhgis_tps.nhgis_dataset_id.nil?       #get out when the terrapop sample is NOT related to an NHGIS dataset
+
+    $stderr.puts "----------------------------------------------------------------------------"
+    $stderr.puts "#{nhgis_tps.country.full_name} => #{sgl.country_level.geog_unit.code} ==> 'GISJOIN'"
+    $stderr.puts "----------------------------------------------------------------------------"
+
+    # the insert statement for the national level is simpler than others, so if parent-level is nil.
+    # check that we're doing national.
+    if parent_sgl.nil?
+      if sgl.internal_code.rindex('NAT').nil? and sgl.internal_code.rindex('FLAD').nil?
+        raise "attempt to call construct_instances_for_terrapop_sample with no parent_level: #{sgl.internal_code}"
+      end
+    end
+
+    (schema, table) = self.shape_table_name.split(".", 2)
+
+    columns_stmt = "SELECT column_name FROM information_schema.columns where table_schema = '#{schema}' AND table_name = '#{table}'"
+
+    columns_records = ActiveRecord::Base.connection.execute(columns_stmt).map{|c| c['column_name']}.uniq
+
+    # once there's a shape_area column on all the maps, we can populate that from here
+    join_clause = ""
+    if !sgl.internal_code.rindex("NAT").nil?                        #nation-level GISJOIN code format varies: e.g. "G1," "G010" depending on the year
+      label_column_source = "'United States'"
+
+    elsif !sgl.internal_code.rindex("FLAD").nil?                    #state-level GISJOIN code format: "G" + "SSS" -- 3 of which is numeric
+
+      potential_names = ['statenam', 'name10', 'name']
+
+      overlap = columns_records & potential_names
+
+      if overlap.count == 0
+        raise "'potential_names' does not contain an overlap with '#{columns_records.join(", ")}' - #{table}"
+      end
+
+      label_column_source = "shape.#{overlap.first}"                        #in NHGIS, states do not have parents
+
+    elsif !sgl.internal_code.rindex("SLAD").nil?
+      #when working with a county, the GISJOIN code will be 8 characters long: "G" + "SSS" + "CCCC" -- 7 of which is numeric
+
+      potential_names = ['nhgisnam', 'name10', 'name']
+
+      overlap = columns_records & potential_names
+
+      if overlap.count == 0
+        raise "'potential_names' does not contain an overlap with '#{columns_records.join(", ")}' - #{table}"
+      end
+
+      label_column_source = "shape.#{overlap.first}"
+
+      join_clause = "INNER JOIN geog_instances parent ON parent.sample_geog_level_id = #{parent_sgl.id} AND parent.code = CAST(SUBSTR(shape.gisjoin,2,3) AS NUMERIC)"
+
+    else
+      raise "SampleGeogLevel was neither 'NAT', 'FLAD', nor 'SLAD' -- unable to generate an select/insert statement for #{self.source_file}"
+    end
+
+    parent_id_column_source = join_clause.empty? ? "NULL" : "parent.id"
+
+    #the select statement will serve as the select for the bulk insert.  The select statement could also be used for debugging purposes
+    select_stmt = "SELECT #{sgl.id}                            AS sample_geog_level_id,
+                             #{parent_id_column_source}        AS parent_id,
+                             #{self.gisjoin_as_numeric_clause} AS code,
+                             #{label_column_source}            AS label,
+                             ST_Area(shape.geog)               AS shape_area,
+                             'GISJOIN'                         AS geog_code,
+                             shape.gisjoin                     AS str_code,
+                             #{self.terrapop_sample_id}        AS terrapop_sample_id
+                        FROM #{self.shape_table_name} shape
+                       #{join_clause}
+                        WHERE shape.gisjoin <> ''"
+
+    insert_stmt = "INSERT INTO geog_instances(sample_geog_level_id, parent_id, code, label, shape_area, geog_code, str_code, terrapop_sample_id)
+                      #{select_stmt}"
+    begin
+      $stderr.puts "inserting geog instances...".color(:magenta)
+      $stderr.puts insert_stmt
+      #p ActiveRecord::Base.connection.exec_query(select_stmt)
+      records = ActiveRecord::Base.connection.insert(insert_stmt)
+    rescue Exception => e
+      raise e.message.color(:red)
+    end
+  end
+
+  def construct_boundaries_for_nhgis_terrapop_sample(sgl)
+
+    #select_stmt provides the values to be inserted into boundaries
+    select_stmt = "SELECT #{self.id} AS map_id,
+                          gi.id      AS geog_instance_id,
+                          gi.code    AS code,
+                          gi.label   AS description,
+                          shape.geog AS geog,
+                          shape.geom AS geom
+                    FROM #{self.shape_table_name} shape
+                    INNER JOIN geog_instances gi ON gi.sample_geog_level_id = #{sgl.id} AND gi.code = #{self.gisjoin_as_numeric_clause}
+                    WHERE shape.gisjoin <> ''"
+
+    #prepare the insert statement using the select statement above
+    insert_stmt = "INSERT INTO boundaries(map_id, geog_instance_id, code, description, geog, geom)
+                    #{select_stmt}"
+
+    #execute the insert statement and catch any errors presenting them to the user.  NOTE: duplicates are not checked and will be added when run twice.
+    begin
+      $stderr.puts "inserting boundaries...".color(:magenta)
+      puts insert_stmt
+      #p ActiveRecord::Base.connection.exec_query(select_stmt)            #slow: the selects can take a while with the geometry and all...
+      records = ActiveRecord::Base.connection.insert(insert_stmt)
+    rescue Exception => e
+      $stderr.puts e.message.color(:red)
+    end
+
+  end
+
+  def shape_table_name
+    #using the map's source file, generate the table name for the corresponding shape file table.
+    Rails.configuration.database_configuration[Rails.env]['gis_schema_name'] + '.' + self.source_file
+  end
+
+  def gisjoin_as_numeric_clause
+    "CAST(SUBSTR(shape.gisjoin,2,LENGTH(shape.gisjoin)-1) AS NUMERIC)"   #same for all levels, just abstracting away complexity
+  end
+
+end