terrapop_models 0.1.0

data/app/models/map_unit.rb

@@ -0,0 +1,4 @@
+class MapUnit < ActiveRecord::Base
+  has_many :map_unit_raster_datasets
+  has_many :raster_datasets, through: :map_unit_raster_datasets
+end

data/app/models/map_unit_raster_dataset.rb

@@ -0,0 +1,4 @@
+class MapUnitRasterDataset < ActiveRecord::Base
+  belongs_to :map_unit
+  belongs_to :raster_dataset
+end

data/app/models/markup_transform.rb

@@ -0,0 +1,240 @@
+# encoding: UTF-8
+
+module MarkupTransform
+
+  def transform_markup(node)
+
+    # special functions
+    # PAGE_NUMBER_TAG = "pg";
+    # FOOTNOTE_TAG = "fn";
+    # FOOTNOTE_REF_TAG = "fnarg";
+    # COMMENT = "comment";
+
+    #insert_image is usa only.
+    #insert_html example: ipumsi educkg
+    #crvar is common in ihis
+
+    # Whether a paragraph tag has been opened
+    @markup_p_open = false
+    @show_hider_count = 0
+
+    return "" if node.nil? || node.empty?
+    node.gsub!(/\r\n?/,"\n")
+    doc = IpumsXmlParsing.build_xml_doc_from_fragment(node)
+    "<p>#{parse_node(doc, default_text_handler)}</p>".html_safe
+  end
+
+  def default_text_handler
+    Proc.new { |item| parse_text(item)}
+  end
+
+  def parse_node (node, text_handler)
+    if node.children.nil?
+      Rails.logger.warn "Node #{node.name} has no children"
+      return ''
+    end
+
+    # svar isn't a formatting tag but includes an opening '\n'; get rid of it
+    node.inner_html = node.inner_html.sub(/\n/, '') if node.name == 'svar'
+
+    node.children.map do |e|
+      e.text? ? text_handler.call(e) : parse_non_text_node(e, text_handler)
+    end.join
+  end
+
+  def parse_non_text_node(e, text_handler)
+    case e.name
+      when 'insert_html'
+        load_html_fragment(e[:id])
+      when 'sample'
+        ''
+      when 'survey_section'
+        ''
+      when 'i1', 'i2', 'i3', 'title' # block styles
+        "<div class=\"#{e.name}\">" + parse_node(e, text_handler) + "</div>"
+      when 'b1', 'b2', 'b3' # list styles
+        "<ul class=\"#{e.name}\">\n" + parse_list(e) + "\n</ul>"
+      when 'em', 'pg', 'ital', 'lang', 'h1', 'h2', 'h3'  # inline styles
+        "<span class=\"#{e.name}\">" + parse_node(e, text_handler) + "</span>"
+      when 'insert_image'
+        load_image(e[:id])
+      when 'link'
+        load_link(e[:id], e.inner_html)
+      when 'svar', 'invar'
+        link_target(e[:a],e[:v]) + parse_node(e, text_handler)
+      when 'crvar'
+        #"<a href=\"#{variable_path(e.inner_html.strip)}\">#{e.inner_html.strip}</a>"
+        variable_path(e.inner_html.strip)
+      when 'crcode'
+        "<a href=\"#{codes_variable_path(e.inner_html.strip)}\">#{e.inner_html.strip}</a>"
+      when 'code'
+        "<div class=\"code\">Codes</div>"
+      when 'topcode'
+        "<div class=\"topcode\">Top codes:</div>"
+      when 'bottomcode'
+        "<div class=\"bottomcode\">Bottom codes:</div>"
+      when 'more'
+        more(e, text_handler)
+      else
+        attrs = e.attributes.map { |name, attr| name + (attr.value.empty? ? "" : "=\"#{attr.value}\"")}.join(" ")
+        "<#{e.name} #{attrs}>" + parse_node(e, text_handler)  + "</#{e.name}>"
+    end
+  end
+
+  def parse_list (node)
+    out = node.children.map do |e|
+  #    puts "Children are: #{e.inspect}"
+      # Split lines into li's
+       (e.text? ? parse_list_text(e) : parse_non_text_node(e, Proc.new {|item| parse_list_text(item) }))
+    end.join
+    out.gsub!(/\A\s*/, '')
+    "<li>" + out +"</li>"
+  end
+
+  def parse_list_text (node)
+    str = node.to_s
+    str.gsub!(/\A\s*\z/, '') ## remove lines that are entirely whitespace
+    str.gsub!(/\A(\s)?\s*/,'\1') ## Remove duplicate whitespace at the beginning
+    str.gsub!(/\n\n+\s*/, "</li>\n\n<li>") ## break 2+ new lines into separate bullets, remove any extra whitespace
+    #puts "Ret is #{str.inspect}"
+    str
+  end
+
+  def parse_text (e)
+    ### Svar is an enclosing tag, but we're moving it's contents outside the <a> tag for display.
+    if e.parent.nil? || e.parent.xml? || e.parent.fragment? || e.parent.name == 'svar' || e.parent.name == 'more'
+      format_in_paragraph e
+    else
+      format_not_in_paragraph e  ### This is already inside some kind of html (e.g. <div> or <ul>.)  We don't want paragraphs.
+    end
+  end
+
+  def format_in_paragraph(e)
+    text = e.to_s
+    # puts "parsing text in paragraph '#{e.to_s}'"
+
+    ## Remove leading double lines if the preceding tag was a block element
+    previous_node = e.previous
+    if block_element?(previous_node)
+      text.gsub!(/\A\n+/m, '')
+    end
+
+    next_node = e.next
+
+    # Remove leading newline if we are in a <more> tag
+    text.gsub!(/\A\n/, '') if e.parent.name == 'more'
+
+    # Remove trailing double line
+    if block_element?(next_node)
+      text.gsub!(/\n+\z/m, '')
+    end
+
+    start_tag = '<p class="paragraph">' # starting tag
+    text = text.to_s.dup
+    if text.gsub!(/\n\n+/, "</p>\n\n#{start_tag}")  # 2+ newline  -> paragraph
+      @markup_p_open = true
+    end
+
+    text.gsub!(/((\A|[^\n])\n)((?=[^\n]|\z))/, '\1<br />')  # 1 newline   -> br
+
+    #prev_node was a block node (ul, div, then start this <p>)
+    if block_element?(previous_node) && !text.empty?
+      text.insert 0, start_tag
+      # valid because we could not have closed in text before this line.  If the logic changes above,
+      #  then this may need to chagne as well
+      @markup_p_open = true
+    end
+
+    #next_node is a block node (ul, div, then close this <p>)
+    if block_element?(next_node) && !text.empty?
+      text << "</p>\n"
+      @markup_p_open = false
+    end
+
+    text
+  end
+
+  def block_element?(node)
+    node && ['more', 'b1', 'b2', 'b3', 'li', 'i1', 'i2', 'i3', 'code', 'topcode', 'bottomcode'].include?(node.name)
+  end
+
+  def inside_ul?(node)
+    node && node.parent && node.parent.name == 'ul'
+  end
+
+  def format_not_in_paragraph(e)
+    # puts "parsing text NOT in paragraph '#{e.to_s}'"
+    # puts "parent is #{e.parent.name}"
+    text = e.to_s
+    text.gsub!(/\n\n+/, "<br /><br />\n\n") unless inside_ul?(e) #2+ newline -> br br
+    text.gsub!(/([^\n]\n)(?=[^\n])/, '\1<br />') # 1 newline   -> br
+    #text = "\n<br />" if text == "\n" # nothing but a single newline -> br
+    text
+  end
+
+  def link_target (anchors, vars)
+    unless anchors.nil?
+      vars_to_anchor = anchors.downcase == "all" ? vars.split(/\s/) : anchors.split(/\s/)
+      return vars_to_anchor.map{|v| "<a name=\"#{v}\"></a>"}.join('')
+    else
+      return ""
+    end
+  end
+
+  def load_file(id)
+    raise "Unused! Use load_html_fragment instead."
+    return '' if Link.hash.empty?
+    dir = IPUMS::project.config[:included_html_dir]
+    path = dir + Link.hash[id]
+    File.open(path).read
+  end
+
+  def load_image(id)
+    href = Link.hash ? Link.hash[id] : ''
+    "<img src=\"#{href}\"/>"
+  end
+
+  # Should replace load_file()
+  def load_html_fragment(link_id)
+    name = Link.hash[link_id]
+    fragment = InsertHtmlFragment.find_by_name(name)
+    if fragment.nil?
+      raise "Could not find HTML fragment link ID #{link_id} with name #{name}."
+    end
+
+    fragment.content
+  end
+
+  def load_link(id, text)
+    "<a href=\"#{Link.hash.empty? ? '' : Link.hash[id]}\">#{text}</a>"
+  end
+
+  def ensure_p_closed
+    if @markup_p_open
+      @markup_p_open = false
+      return '</p>'
+    end
+    ''
+  end
+
+  def open_p
+    @markup_p_open = true
+    '<p>'
+  end
+
+  def more(e, text_handler)
+    @show_hider_count += 1
+    p_was_open = @markup_p_open
+    result = "<a id=\"toggler-#{@show_hider_count}\" title=\"Toggle visibility\" class=\"show-hide-toggler\">&nbsp;</a>"
+    result += ensure_p_closed
+    result += "<div id=\"toggler-#{@show_hider_count}-target\">"
+    result += open_p
+    result += parse_node(e, text_handler)
+    result += ensure_p_closed
+    result += "</div><p></p>"
+    if p_was_open
+      result += open_p
+    end
+    result
+  end
+end

data/app/models/measurement_type.rb

@@ -0,0 +1,5 @@
+class MeasurementType < ActiveRecord::Base
+
+  has_many :area_data_variables
+
+end

data/app/models/nhgis/agg_data_var.rb

@@ -0,0 +1,120 @@
+require 'nhgis_database'
+module Nhgis
+  class AggDataVar < NhgisActiveRecord::Base
+    belongs_to :data_table
+    
+    PAGE_SIZE = 1000
+    
+    def self.variables_count(dataset_id)
+      
+      sql =  'select
+                COUNT(ds.code) AS count
+              from agg_data_vars adv
+              join data_tables dt on dt.id = adv.data_table_id
+              join datasets ds on ds.id = dt.dataset_id and ds.id = ?
+              join data_groups dg on dg.dataset_id = dt.dataset_id AND dg.relative_pathname IS NOT NULL
+              join geotimes gt on gt.id = dg.geotime_id
+              join geog_levels gl on gl.id = gt.geog_level_id and gl.istads_id IN ("state", "county", "nation")
+              join data_files df on df.data_group_id = dg.id
+              join time_series_components tsc on tsc.agg_data_var_id = adv.id
+              join time_series ts on ts.id= tsc.time_series_id
+              join time_series_tables_x_time_series x on x.time_series_id = ts.id
+              join time_series_tables tst on tst.id = x.time_series_table_id'
+      
+      sql = send(:sanitize_sql_array, [sql, dataset_id])
+      result = connection.execute(sql)
+      
+      count = result[0]['count'].to_i
+    end
+    
+    def self.variables(dataset_id, page = -1)
+            
+      sql =  "select
+                ds.code as ds_code,
+                dt.label as dt_label,
+                adv.label as adv_label,
+                adv.id as adv_id,
+                ts.id as ts_id,
+                ts.label as ts_label,
+                tst.id as tst_id,
+                tst.label as tst_label,
+                gl.id as gl_id,
+                gl.istads_id as gl_istads_id,
+                dg.id as dg_id,
+                dg.relative_pathname as dg_relative_pathname,
+                df.id as df_id,
+                df.filename as df_filename,
+                ds.id as ds_id
+              from agg_data_vars adv
+              join data_tables dt on dt.id = adv.data_table_id
+              join datasets ds on ds.id = dt.dataset_id and ds.id = ?
+              join data_groups dg on dg.dataset_id = dt.dataset_id AND dg.relative_pathname IS NOT NULL
+              join geotimes gt on gt.id = dg.geotime_id
+              join geog_levels gl on gl.id = gt.geog_level_id and gl.istads_id IN ('state', 'county', 'nation')
+              join data_files df on df.data_group_id = dg.id
+              join time_series_components tsc on tsc.agg_data_var_id = adv.id
+              join time_series ts on ts.id= tsc.time_series_id
+              join time_series_tables_x_time_series x on x.time_series_id = ts.id
+              join time_series_tables tst on tst.id = x.time_series_table_id LIMIT #{page},#{PAGE_SIZE}"
+              
+              #order by adv.istads_seq'
+      
+      #connection.execute([sanitize(sql), dataset_id, geog_level_id])
+      sql = send(:sanitize_sql_array, [sql, dataset_id])
+      connection.execute(sql)
+    end
+    
+    def self.all_variables(limit_dataset_ids = nil)
+      
+      dataset_ids = limit_dataset_ids.nil? ? Dataset.where({}).select("id") : Dataset.where({id: (limit_dataset_ids.is_a?(Array) ? limit_dataset_ids : [limit_dataset_ids])}).select("id")
+      
+      file = File.join(TerrapopConfiguration['application']['environments'][Rails.env.to_s]['source_data']['nhgis_metadata'], 'nhgis-agg-data-vars.csv.gz')
+      
+      
+      File.open(file, 'w') do |f|
+        gz = Zlib::GzipWriter.new(f)
+
+        csv = CSV.new(gz)
+        
+        variables = Nhgis::AggDataVar.variables(dataset_ids.first, 0)
+        columns = variables.first.keys
+        csv << columns
+      
+        csv.close
+      
+        gz.close
+      end
+      
+      dataset_ids.each{|dataset|
+        
+        $stderr.print "===> Working on dataset[#{dataset.id}]..."
+        
+        count = AggDataVar.variables_count(dataset.id)
+        
+        pages = (count.to_f/PAGE_SIZE.to_f).ceil
+
+        (0...pages).each{|page|
+          variables = AggDataVar.variables(dataset.id, page)
+          
+          File.open(file, 'a+') do |f|
+            gz = Zlib::GzipWriter.new(f)
+            csv = CSV.new(gz)
+          
+            variables.each {|row|
+              csv << row.values
+            }
+          
+            csv.close
+            gz.close
+          end
+        }
+        
+        $stderr.puts "done"
+      }
+      
+      $stderr.puts "==> Done with datasets..."
+      
+    end
+    
+  end
+end

data/app/models/nhgis/breakdown_combo.rb

@@ -0,0 +1,84 @@
+require 'nhgis_database'
+module Nhgis
+  class BreakdownCombo < NhgisActiveRecord::Base
+
+    has_many :data_files
+    has_many :time_series_components
+    has_many :breakdown_value_breakdown_combos
+    has_many :breakdown_values,:through=>:breakdown_value_breakdown_combos
+  
+    def self.istads_ids_for_sql(breakdown_value_istads_id_list)
+      #this method will convert an array of breakdown value istads id into a string that can be used in a sql statement
+      breakdown_value_istads_id_list.inspect.gsub("[", "").gsub("]", "").gsub("\"", "'")
+    end
+
+    def self.defaults
+      sql =<<-EOS
+          SELECT bc.*
+          FROM breakdown_combos bc
+          JOIN breakdown_values_x_breakdown_combos x ON x.breakdown_combo_id = bc.id
+          JOIN breakdown_values bv                   ON bv.id = x.breakdown_value_id
+          GROUP BY bc.id HAVING COUNT(bv.id) = SUM(is_default);
+      EOS
+      find_by_sql(sql)
+    end
+
+    def self.find_by_breakdown_values(breakdown_value_istads_ids)
+      # there is an assumption that breakdown_value_istads_ids will contain the istads ids belonging to...
+      #    NO breakdown value set (when empty),
+      #    one breakdown value set (when breakdown value istads ids begin with the SAME prefix e.g. "bs01"
+      #    two breakdown value sets (when breakdown value istads ids begin with one prefix or another e.g. "bs01" or "bs02"
+      # ASSUMPTION is that there is never a valid breakdown combo where
+
+      #puts "*** In find_by_breakdown_values #{breakdown_value_istads_ids.inspect}"
+      breakdown_value_sets = breakdown_value_istads_ids.map{|bv|
+      bv.
+      split(".").
+      first}.flatten.uniq
+    
+      case breakdown_value_sets.size
+        when 0
+          return []
+        when 1
+          sql_values = BreakdownCombo.istads_ids_for_sql(breakdown_value_istads_ids) 
+          sql = "SELECT DISTINCT * FROM breakdown_combos WHERE istads_id IN (#{sql_values})"
+        when 2
+          bs1 = breakdown_value_sets.first
+          bs2 = breakdown_value_sets.last
+          bvs1 = breakdown_value_istads_ids.select{|bv| bv.split(".").first == bs1}
+          bvs2 = breakdown_value_istads_ids.select{|bv| bv.split(".").first == bs2}
+          sql_values1 = BreakdownCombo.istads_ids_for_sql(bvs1)
+          sql_values2 = BreakdownCombo.istads_ids_for_sql(bvs2)
+
+          sql = "SELECT DISTINCT bc.* FROM breakdown_combos bc "
+          sql << "JOIN breakdown_values_x_breakdown_combos x ON x.breakdown_combo_id = bc.id "
+          sql << "JOIN breakdown_values bv ON bv.id = x.breakdown_value_id AND bv.istads_id LIKE '#{bs1}.%' "
+          sql << "JOIN breakdown_values_x_breakdown_combos x2 ON x2.breakdown_combo_id = x.breakdown_combo_id "
+          sql << "JOIN breakdown_values bv2 ON bv2.id = x2.breakdown_value_id AND bv2.istads_id LIKE '#{bs2}.%' "
+          sql << "WHERE bv.istads_id IN (#{sql_values1}) AND bv2.istads_id IN (#{sql_values2})"
+        else
+          raise "too many breakdown value sets; only expected 1 or 2 (or none)"
+      end
+      find_by_sql(sql)
+    end
+
+    def codebook_labels
+      sorted_breakdown_values.map{|bv| bv.composite_codebook_label}
+    end
+
+    def header_label
+      #"[breakdown_var 1 value]: [breakdown_var 2 value]"
+      # NOTE : currently sorting the breakdowns using the breakdown value set istads_id
+      sorted_breakdown_values.map{|bv| bv.label}.join(": ")
+    end
+
+    def is_default?
+      breakdown_values.each{|bv| return false unless bv.is_default}
+      return true
+    end
+
+    def sorted_breakdown_values
+      breakdown_values.sort{|a,b| a.breakdown_value_set.istads_seq <=> b.breakdown_value_set.istads_seq}
+    end
+  end
+end