terrapop_models 0.1.0

data/app/models/error_event.rb

@@ -0,0 +1,2 @@
+class ErrorEvent < ActiveRecord::Base
+end

data/app/models/extract_data_artifact.rb

@@ -0,0 +1,4 @@
+class ExtractDataArtifact < ActiveRecord::Base
+  belongs_to :extract_request
+  store_accessor :variables_description
+end

data/app/models/extract_request.rb

@@ -0,0 +1,817 @@
+require 'extract_type'
+require 'chronic_duration'
+
+class ExtractRequest < ActiveRecord::Base
+  include ExtractEngine::FileManagement
+  include ActiveModel::Validations
+
+  class ExtractTypeConst
+    # see data/static_data.yml for the available status_definitions
+    # [:RASTER, :AREA, :UNKNOWN, :MICRODATA]
+
+    #def self.const_missing(name)
+      #abort("Unable to find [#{name.to_s}]")
+    #end
+
+    def self.define_consts
+      @@extract_types = ExtractType.all
+      # magic constant injections!
+      @@extract_types.each do |extract_type|
+        label = extract_type.label
+        raise "Your database maybe in an unknown state; ExtractType's label is nil" if label.nil?
+        label = label.downcase.gsub(/ /, '_')
+        self.const_set(label.upcase, label)
+        parent.scope(label, -> { parent.where(status: label) })
+      end
+    end
+
+
+    def self.const_missing(name)
+      name.downcase.to_s
+    end
+
+  end
+
+
+  class NewuiExtractValidator < ActiveModel::Validator
+
+    def validate(record)
+      record.errors[:base] << 'Undefined User' if record.user.blank?
+      record.errors[:base] << 'Undefined Extract' if record.data.blank?
+      record.errors[:base] << 'Unknown Extract Type' if record.data["type"].blank?
+      record.errors[:base] << 'Wrong Extract Type' unless ["arealevel", "microdata", "raster"].include?(record.data["type"])
+      record.errors[:base] << 'Unknown Extract Title' if record.data["title"].blank?
+      record.errors[:base] << "Wrong value in 'Include boundary files'" unless !!record.data["boundary"] == record.data["boundary"]
+      record.errors[:base] << "Wrong value in 'Send Extract to data grid'" unless !!record.data["datagrid"] == record.data["datagrid"]
+      case record.data["type"]
+      when "microdata"
+      when "arealevel"
+        if record.data["arealevel_variables"].empty? && record.data["arealevel_datasets"].empty?
+          if record.data["raster_variables"].empty? && record.data["raster_datasets"].empty?
+            record.errors[:base] << "Both 'Area-Level Data' and 'Raster Data' are empty"
+          else
+            record.errors[:base] << "Missing 'Georgraphic Level' for 'Raster Data'" if record.data["raster_geographic_levels"].empty?
+            record.errors[:base] << "Missing 'Operations' for 'Raster Data'" if record.data["raster_operations"].empty?
+          end
+        else
+          record.errors[:base] << "Missing 'Georgraphic Level' for 'Area-Level Data'" if record.data["arealevel_geographic_levels"].empty?
+          record.errors[:base] << "Missing 'Operations' for 'Raster Data'" if !(record.data["raster_variables"].empty? || record.data["raster_datasets"].empty?) && record.data["raster_operations"].empty?
+        end
+      when "raster"
+      end
+    end
+
+  end
+
+
+  store_accessor :data
+
+  has_many :extract_statuses, dependent: :destroy
+  has_many :extract_request_submissions, dependent: :destroy
+  # For creating the request incrementally and for reporting, you need these directly
+  has_many :request_area_data_variables, dependent: :destroy
+  # You may pass these to the extract engine, if no extra attributes of the requested variables are required on a per request basis.
+  has_many :area_data_variables, -> { uniq }, through: :request_area_data_variables, dependent: :destroy
+  has_many :request_variables, dependent: :destroy
+  has_many :variables, -> { uniq }, through: :request_variables, dependent: :destroy
+  has_many :request_raster_variables, dependent: :destroy
+  has_many :raster_variables, -> { uniq }, through: :request_raster_variables, dependent: :destroy
+  has_many :request_geog_units, dependent: :destroy
+  has_many :geog_units, -> { uniq }, through: :request_geog_units, dependent: :destroy
+  has_many :request_terrapop_samples, dependent: :destroy
+  has_many :terrapop_samples, -> { uniq }, through: :request_terrapop_samples, dependent: :destroy
+  has_many :request_samples, dependent: :destroy
+  has_many :samples, -> { uniq }, through: :request_samples, dependent: :destroy
+  has_many :request_raster_datasets, dependent: :destroy
+  has_many :raster_datasets, -> { uniq }, through: :request_raster_datasets, dependent: :destroy
+  has_many :request_sample_geog_levels, dependent: :destroy
+  has_many :sample_geog_levels, -> { order(id: :desc).uniq }, through: :request_sample_geog_levels, dependent: :destroy
+  has_many :extract_request_error_events, dependent: :destroy
+  has_many :error_events, through: :extract_request_error_events, dependent: :destroy
+  has_many :derivatives, class_name: "ExtractRequest", foreign_key: "revision_of_id", dependent: :nullify
+  has_many :extract_request_area_data_raster_variable_mnemonic_lookups, dependent: :destroy
+  has_many :area_data_raster_variable_mnemonic_lookups, through: :extract_request_area_data_raster_variable_mnemonic_lookups, dependent: :destroy
+
+  has_and_belongs_to_many :labels, join_table: "extract_requests_labels"
+
+  has_many :extract_data_artifacts
+
+  belongs_to :user
+  belongs_to :derivative, class_name: "ExtractRequest", foreign_key: "revision_of_id"
+
+  scope :submissions, -> { where(submitted: true) }
+  scope :unsubmitted, -> { where(submitted: false) }
+  scope :recent, -> { where("created_at > ?", Time.now - 7.day) }
+  scope :queued, -> { where(finish_extract_time: nil).joins(:extract_statuses).where(extract_statuses: {status: 'enqueued'}) }
+  scope :get_user_extracts, ->(user_id) { where(user_id: user_id) }
+  scope :get_submitted_user_extracts, ->(user_id) { distinct.where(user_id: user_id).joins(:extract_statuses).where.not(extract_statuses: {status: 'building_request'}) }
+
+  validates_with NewuiExtractValidator, on: :create, if: :from_newui?
+
+  after_create :normalize_attributes
+
+
+  def self.dump_all
+    {
+      self.to_s.underscore.pluralize => self.where.not(user: nil).find_each.map { |extract|
+        micro_variable_mnemonics  = extract.variables.select(:mnemonic).map(&:mnemonic)
+        area_variable_mnemonics   = extract.area_data_variables.select(:mnemonic).map(&:mnemonic)
+        raster_dataset_mnemonics  = extract.raster_datasets.select(:mnemonic).map(&:mnemonic)
+        terrapop_sample_mnemonics = extract.terrapop_samples.select(:label).map(&:label)
+        sample_names              = extract.samples.select(:name).map(&:name)
+        {
+          'variable_ids'           => micro_variable_mnemonics,
+          'area_data_variable_ids' => area_variable_mnemonics,
+          'raster_dataset_ids'     => raster_dataset_mnemonics,
+          'terrapop_sample_ids'    => terrapop_sample_mnemonics,
+          'sample_ids'             => sample_names,
+          'boundary_files'         => extract.boundary_files,
+          'title'                  => extract.title,
+          'notes'                  => extract.notes,
+          'user_id'                => extract.user.email,
+          'uuid'                   => extract.uuid,
+          'submitted'              => extract.submitted,
+          'created_at'             => extract.created_at,
+          'updated_at'             => extract.updated_at,
+         #'derivative'             => extract.uuid,
+          'file_type'              => extract.file_type,
+          'raster_only'            => extract.raster_only,
+          'send_to_irods'          => extract.send_to_irods,
+          'extract_url'            => extract.extract_url,
+          'request_url'            => extract.request_url,
+          'begin_extract_time'     => extract.begin_extract_time,
+          'finish_extract_time'    => extract.finish_extract_time,
+          'total_time'             => extract.total_time
+        }
+      },
+      'request_raster_variables' => RequestRasterVariable.find_each.map { |request_variable|
+        unless request_variable.raster_operation.nil?
+          unless request_variable.extract_request.user.nil?
+            {
+              'extract_request_id'  => request_variable.extract_request.uuid,
+              'raster_variable_id'  => request_variable.raster_variable.mnemonic,
+              'raster_operation_id' => request_variable.raster_operation.name
+            }
+          end
+        end
+      }.compact
+    }
+  end
+
+
+  def from_newui?
+    self.origin == 'newui'
+  end
+
+
+  def most_recent_submitted_at
+    mrsa = self.extract_request_submissions.most_recent.first
+    mrsa.submitted_at unless mrsa.nil?
+  end
+
+
+  def hierarchical?
+    self.file_type == "hierarchical"
+  end
+
+
+  def name
+    "terrapop_extract_#{id.to_s}"
+  end
+
+
+  # Public path in URL, can't get from terrapop.yml
+  def extract_file_path(ext = nil)
+    path = File.join(location, name)
+    path += ".#{ext}" unless ext.nil?
+    path
+  end
+
+
+  # Public path in URL, can't get from Terrapop.yml
+  def download_path
+    #"/extracts/#{user_directory()}/#{extract_directory()}"
+    File.join('/extracts', user_directory, extract_directory)
+  end
+
+
+  def boundary_file_stub
+    "boundaryfiles_#{id.to_s}"
+  end
+
+
+  def boundary_file_name(sample_geog_level)
+    "#{boundary_file_stub}_#{sample_geog_level.internal_code.to_s}"
+  end
+
+
+  def boundary_file_names
+    sample_geog_levels.map{ |sgl| boundary_file_name(sgl) }
+  end
+
+
+  def boundary_file_bundle
+    "#{boundary_file_stub}.zip"
+  end
+
+
+  def boundary_file_bundle_path
+    File.join(location, boundary_file_bundle)
+  end
+  
+  def data_file_summaries
+    str = []
+    unless extract_data_artifacts.count == 0
+      
+      extract_data_artifacts.each do |eda|
+        str << "Data File: " + eda.variables_description[:filename] 
+        if boundary_files
+          str << "Boundary Shapefile: " + eda.variables_description[:boundary_filename] 
+        end
+        str << "Country: " + eda.variables_description[:country]
+        str << "Data Year(s): " + eda.variables_description[:years]
+        str << "Geographic Level: " + eda.variables_description[:geog_unit] + ": " + eda.variables_description[:country_level]
+        str << ""
+      end
+      
+    end
+    str.join("\n")
+    
+  end
+
+  def data_dictionaries
+    str = []
+    unless extract_data_artifacts.count == 0
+      extract_data_artifacts.each do |eda|
+        str << "Data File: " + eda.variables_description[:filename] 
+        eda.variables_description[:columns].each do |column|
+          adrvml = AreaDataRasterVariableMnemonicLookup.where(composite_mnemonic: column).first
+          str << column.to_s + ": " + (adrvml.nil? ? "N/A" : adrvml.description)
+        end
+        str << ""
+      end
+    end
+    str.join("\n")
+  end
+
+  # Need an area data variable for every sample_geog_level+ area data variable combination.
+  def expanded_request_area_data_variables
+    request_area_data_variables.map { |variable|
+      sample_geog_levels.map do |sg|
+        request_variable = RequestAreaDataVariable.new
+        request_variable.area_data_variable = variable.area_data_variable
+        request_variable.extract_request = variable.extract_request
+        request_variable.sample_geog_level = sg
+        request_variable
+      end
+    }.flatten
+  end
+
+
+  def expanded_request_raster_variables
+    # .reject{|v| v.raster_operation_id.nil? or v.raster_dataset_id.nil? }
+    request_raster_variables.map { |variable|
+      sample_geog_levels.map do |sg|
+        request_variable = RequestRasterVariable.new
+        #request_variable.raster_variable_id   = variable.raster_variable.id
+        #request_variable.raster_operation_id  = variable.raster_operation.id
+        #request_variable.extract_request_id   = variable.extract_request.id
+        #request_variable.raster_dataset_id    = variable.raster_dataset.id
+        #request_variable.sample_geog_level_id = sg.id
+        request_variable.raster_variable   = variable.raster_variable
+        request_variable.raster_operation  = variable.raster_operation
+        request_variable.extract_request   = variable.extract_request
+        request_variable.raster_dataset    = variable.raster_dataset
+        request_variable.sample_geog_level = sg
+        request_variable
+      end
+    }.flatten
+    #puts "\n\nexpanded_request_raster_variables: \n"
+    #puts "#{rrvs.map{|rrv| rrv.inspect}.join("\n")}"
+    #rrvs
+  end
+
+  def has_area_data_variables?
+    area_data_variables.count > 0
+  end
+
+
+  def has_microdata_variables?
+    variables.count > 0
+  end
+
+
+  def has_raster_variables?
+    raster_variables.count > 0
+  end
+
+
+  def has_variables?
+    has_area_data_variables? || has_microdata_variables? || has_raster_variables?
+  end
+
+
+  def is_empty?
+    !has_variables?
+  end
+
+
+  def variables_count
+    area_data_variables.count + variables.count + raster_variables.count
+  end
+
+
+  def datasets_count
+    terrapop_samples.count + samples.count + raster_datasets.count
+  end
+
+
+  def has_area_data_variable?
+    area_data_variables.exists?(mnemonic: mnemonic)
+  end
+
+
+  def has_microdata_variable?(mnemonic)
+    variables.exists?(mnemonic: mnemonic)
+  end
+
+
+  def has_raster_variable?(mnemonic)
+    raster_variables.exists?(mnemonic: mnemonic)
+  end
+
+
+  def has_variable?(mnemonic, type)
+    case type
+    when ExtractTypeConst::AGGREGATE
+      has_area_data_variable?(mnemonic)
+    when ExtractTypeConst::MICRODATA
+      has_microdata_variable?(mnemonic)
+    when ExtractTypeConst::RASTER
+      has_raster_variable?(mnemonic)
+    else
+      false
+    end
+  end
+
+
+  def extract_type
+    if (has_raster_variables? || has_area_data_variables?) && has_microdata_variables?
+      ExtractTypeConst::AGGREGATE_ATTACHED_TO_MICRODATA
+    elsif has_microdata_variables?
+      ExtractTypeConst::MICRODATA
+    elsif has_area_data_variables? && !raster_only
+      ExtractTypeConst::AGGREGATE
+    elsif has_area_data_variables? && raster_only
+      ExtractTypeConst::RASTER
+    elsif has_raster_variables? && !raster_only
+      ExtractTypeConst::AGGREGATE
+    elsif has_raster_variables? && raster_only
+      ExtractTypeConst::RASTER
+    else
+      ExtractTypeConst::NO_DATA
+    end
+  end
+
+
+  def submit
+    # Changing the submitted setting first captures the idea that the extract is intended to be created and the user wants the data
+    # requested. This will be set even if the documentation / syntax file step or the
+    # enqueueing step fails.
+    #update_attribute(:processing, false)
+    update_attribute(:submitted_at, DateTime.now())
+    update_attribute(:submitted, true)
+    
+    begin
+      tp_web_build = Rails.configuration.build_number
+      
+      unless tp_web_build.to_s.is_i?
+        tp_web_build = -1
+      end
+      
+    rescue 
+      tp_web_build = -1
+    end
+    
+    update_attribute(:tp_web_build_number, tp_web_build)
+    
+    enqueue
+  end
+
+
+  def completed
+    add_status(ExtractStatus::COMPLETED)
+  end
+
+
+  def failed
+    add_status(ExtractStatus::FAILED)
+  end
+
+
+  def enqueue
+    ers = ExtractRequestSubmission.new
+    ers.submitted_at = Time.new
+    ers.save
+    self.extract_request_submissions << ers
+    #update_attribute(:processing, false)
+      
+    add_status(ExtractStatus::ENQUEUED, {processing: false})
+  end
+
+
+  def begin_processing
+    ### This method usually only gets called by the extract engine.
+    
+    add_status(ExtractStatus::PROCESSING)
+    
+    begin
+      tp_xtr_build = Rails.configuration.build_number
+      
+      unless tp_xtr_build.to_s.is_i?
+        tp_xtr_build = -1
+      end
+      
+    rescue 
+      tp_xtr_build = -1
+    end
+    
+    update_attribute(:tp_xtr_build_number, tp_xtr_build)
+    
+  end
+
+
+  def waiting
+    add_status(ExtractStatus::WAITING)
+  end
+
+
+  def stopped
+    add_status(ExtractStatus::STOPPED)
+  end
+
+
+  def email_sent
+    add_status(ExtractStatus::EMAIL_SENT)
+  end
+
+
+  def building_request
+    add_status(ExtractStatus::BUILDING_REQUEST)
+  end
+
+
+  def should_stop
+    add_status(ExtractStatus::SHOULD_STOP)
+  end
+
+
+  def current_status
+    extract_statuses.order(:updated_at).last
+  end
+
+
+  def is_completed?
+    current_status.status == ExtractStatus::COMPLETED
+  end
+
+
+  def is_failed?
+    current_status.status == ExtractStatus::FAILED
+  end
+
+
+  def is_enqueued?
+    current_status.status == ExtractStatus::ENQUEUED
+  end
+
+
+  def is_processing?
+    current_status.status == ExtractStatus::PROCESSING
+  end
+
+
+  def is_waiting?
+    current_status.status == ExtractStatus::WAITING
+  end
+
+
+  def is_stopped?
+    current_status.status == ExtractStatus::STOPPED
+  end
+
+
+  def is_email_sent?
+    current_status.status == ExtractStatus::EMAIL_SENT
+  end
+
+
+  def is_building_request?
+    current_status.status == ExtractStatus::BUILDING_REQUEST
+  end
+
+
+  def should_stop?
+    current_status.status == ExtractStatus::SHOULD_STOP
+  end
+
+
+  def can_be_resubmitted?
+    (is_completed? || is_failed? || is_stopped? || is_email_sent?) || false
+  end
+
+  # This is efficient for getting the current statuses where that status
+  # is "enqueued".  Otherwise we'd have to use includes(:extract_status) on
+  # lots of extract_requests and filter. If this is slow check for indexing on extract_statuses.
+  def self.enqueued
+    ExtractStatus.current.enqueued.extracts.map { |status| status.extract_request }
+  end
+
+  def self.extract_group_count(group_str)
+    ExtractStatus.current.enqueued.extracts.joins("INNER JOIN extract_requests er ON er.id = extract_statuses.extract_request_id").where("extract_requests.extract_grouping" => group_str).count
+  end
+
+  def self.get_next_in_queue
+    ActiveRecord::Base.transaction {
+      extract_ids = ExtractStatus.current.enqueued.extracts.map { |status| status.extract_request_id }
+      found = false
+      e = nil
+      while extract_ids.count > 0 and found == false
+        extract_id = extract_ids.shift
+
+        sql = "SELECT id FROM extract_requests WHERE id = #{extract_id} AND processing = false FOR UPDATE"
+        results = ActiveRecord::Base.connection.execute(sql).first
+        unless results.nil?
+          extract_id = results['id']
+          sql = "UPDATE extract_requests SET processing = true WHERE id = #{extract_id}"
+          ActiveRecord::Base.connection.execute(sql)
+          found = true
+          e = ExtractRequest.find(extract_id)
+        end
+
+      end
+
+      e
+    }
+  end
+
+  def self.num_extracts_in_queue
+    ExtractStatus.current.enqueued.extracts.count
+  end
+
+  def mnemonic(request_variable)
+    use_long_svar_mnemonic?(request_variable.variable) ? request_variable.long_mnemonic : request_variable.mnemonic
+  end
+
+  def use_long_svar_mnemonic?(variable)
+    variable.is_svar? && !variable.long_mnemonic.blank? # && use_long_svar_names?
+  end
+
+  def category_code(request_variable, category)
+    request_variable.general? ? category.code[0..request_variable.width - 1] : category.code
+  end
+
+  def self.duplicate(id_or_extract)
+    if extract = ExtractRequest.find(id_or_extract)
+      new_extract = ExtractRequest.create
+      new_extract.area_data_variables = extract.area_data_variables
+      new_extract.terrapop_samples = extract.terrapop_samples
+      new_extract.variables = extract.variables.where(preselect_status: 0)
+      new_extract.samples = extract.samples
+      extract.request_raster_variables.each do |variable|
+        new_extract.request_raster_variables.create({raster_variable: variable.raster_variable, raster_operation: variable.raster_operation, raster_dataset: variable.raster_dataset})
+      end
+      new_extract.raster_datasets = extract.raster_datasets
+      new_extract.geog_units = extract.geog_units
+      new_extract.sample_geog_levels = extract.sample_geog_levels #quickfix that may have complications!
+      new_extract.boundary_files = extract.boundary_files
+      new_extract.send_to_irods = extract.send_to_irods
+      new_extract.file_type = extract.file_type
+      new_extract.raster_only = extract.raster_only
+      new_extract.title = "Revision of Extract #{extract.id}" + ((!extract.title.blank? && " - #{extract.title}") || '')
+      new_extract.revision_of_id = extract.id
+      new_extract.user_id = extract.user_id
+      new_extract.save
+      new_extract.reload
+    end
+  end
+
+
+  def microdata_size_in_bytes
+    total = 0
+    samples.each do |sample|
+      h_records = sample.h_records
+      p_records = sample.p_records
+      t_h = 0
+      t_p = 0
+      variables.each do |variable|
+        size = variable.column_width
+        if variable.record_type == 'H'
+          t_h += (size * h_records)
+        elsif variable.record_type == 'P'
+          t_p += (size * p_records)
+        end
+      end
+      total += t_h + t_p
+    end
+    total
+  end
+
+
+  def extract_zip_file
+    file = nil
+    if !self.extract_url.nil? && is_completed?
+      begin
+        file = File.join(Rails.root.to_s, "public", URI(self.extract_url).request_uri)
+      rescue Exception => e
+        Rails.logger.debug e
+      end
+    end
+    file
+  end
+
+
+  def extract_zip_exists?
+    file = self.extract_zip_file
+    file.nil? ? false : File.exist?(file)
+  end
+
+
+  def extract_zip_size
+    self.extract_zip_exists? ? File.size(self.extract_zip_file) : nil
+  end
+
+
+  def extract_zip_readable_size
+    self.extract_zip_exists? ? "#{('%.2f' % ((self.extract_zip_size).to_f / 2**20)).to_f} MB" : ''
+  end
+
+
+  # Puts all variables together in a list in the order they should appear in the extract
+  def variables_in_extract
+    layout_variables = variables + expanded_request_area_data_variables + expanded_request_raster_variables
+    if extract_type == ExtractRequest::ExtractTypeConst::AGGREGATE
+      #$stderr.puts "======> sample_geog_levels: " + sample_geog_levels.inspect
+      raise "There were no sample_geog_levels, extract FAILED!!" if sample_geog_levels.first.nil?
+      layout_variables = sample_geog_levels.first.variables_for_area_data_extracts + layout_variables
+    end
+    layout_variables.sort{ |a,b| a.mnemonic<=>b.mnemonic }
+  end
+
+
+  # helper method to get all the years covered by the extract request
+  def years
+    yrs = terrapop_samples.pluck(:year)
+    yrs += samples.pluck(:year)
+    yrs.uniq!
+    years = Hash[ yrs.map { |year| [year, true] } ]
+    years[:count] = yrs.length
+    years
+  end
+
+
+  # helper method to get all the countries covered by the extract request
+  def countries
+    cntries = terrapop_samples.pluck(:short_country_name)
+    cntries += samples.find_each.map { |sample| sample.short_country_name }
+    cntries.uniq!
+    countries = Hash[ cntries.map { |country| [country, true] } ]
+    countries[:count] = cntries.length
+    countries
+  end
+
+
+  def add_preselected_variables
+    if !self.raster_only && self.variables.count > 0
+      preselected_variables = Variable.where(preselect_status: 1).map{ |v| {v => false} }.reduce({}, :merge)
+      self.samples.each do |s|
+        preselected_variables.each do |var, v|
+          if v == false and var.included_in?(s)
+            preselected_variables[var] = true
+          end
+        end
+      end
+      self.variables.concat preselected_variables.reject{ |k, v| v == false }.keys
+    end
+  end
+
+
+  def self.new_extract_grouping
+    str = SecureRandom::hex(8)
+    while ExtractRequest.where(extract_grouping: str).count > 0
+      str = SecureRandom::hex(8)
+    end
+    str
+  end
+
+
+  def import
+    # this method is very cusomized for arealevel extract worflow, more to come
+    self.title = self.data["title"]
+    self.notes = self.data["notes"]
+    self.boundary_files = self.data["boundary"]
+    self.send_to_irods = self.data["datagrid"]
+    self.raster_only = self.data["type"] == "raster"
+    self.file_type = 'csv' #maybe a case on the self.data["type"]
+    #self.variables = coming soon
+    #self.samples = coming soon
+    self.area_data_variables = AreaDataVariable.where(mnemonic: self.data["arealevel_variables"])
+    self.terrapop_samples = TerrapopSample.where(id: self.data["arealevel_datasets"])
+    geographic_level_ids = self.data["arealevel_geographic_levels"].empty? ? (self.data["raster_geographic_levels"].empty? ? [] : self.data["raster_geographic_levels"]) : self.data["arealevel_geographic_levels"]
+    unless geographic_level_ids.empty?
+      self.sample_geog_levels = SampleGeogLevel.where(id: geographic_level_ids)
+      self.geog_units = self.sample_geog_levels.map{ |sgl| sgl.geog_unit }.uniq
+    end
+    self.raster_datasets = RasterDataset.where(mnemonic: self.data["raster_datasets"])
+    self.data["raster_operations"].each do |variable_mnemonic, operation_opcodes|
+      raster_variable = RasterVariable.find_by(mnemonic: variable_mnemonic)
+      unless raster_variable.nil?
+        raster_variable.raster_datasets.where(mnemonic: data["raster_datasets"]).each do |raster_dataset|
+          RasterOperation.where(opcode: operation_opcodes).each do |operation|
+            raster_operations = []
+            raster_operations.push(operation.children.size > 0 ? (operation.children & raster_variable.raster_data_type.raster_operations).first : operation).flatten.uniq!
+            raster_operations.each do |raster_operation|
+              self.request_raster_variables.create({raster_variable: raster_variable, raster_operation: raster_operation, raster_dataset: raster_dataset})
+            end
+          end
+        end
+      end
+    end
+  end
+
+
+  def export
+    {}
+  end
+
+
+  def submitted_date_time
+    unless submitted_at.nil?
+      submitted_at.strftime('%D - %r')
+    else
+      "N/A"
+    end
+  end
+
+
+  def total_time_human_readable
+    
+    if total_time.nil?
+      "- minutes"
+    else
+      ChronicDuration.output((total_time / 1000.0), :format => :long)
+    end
+    
+  end
+  
+  def extract_type_human_readable
+    if extract_type == 'aggregate'
+      'AREA-LEVEL'
+    else
+      extract_type.upcase
+    end
+  end
+
+  def can_be_revised?
+    origin != 'newui'
+  end
+
+
+  def include_boundary_files_human_readable
+    boundary_files ? "Yes" : "No"
+  end
+
+
+  private
+
+
+  def add_status(status_string, extra_fields = {})
+    extract_status = ExtractStatus.new
+    extract_status.status = status_string
+    extract_status.extract_request = self
+    
+    sd = StatusDefinition.where(status: status_string.downcase).first
+    
+    unless sd.nil?
+      extract_status.status_definition_id = sd.id
+    end
+    
+    extract_status.save
+    self.extract_statuses << extract_status
+    
+    extra_fields.each do |k,v|
+      self.send("#{k}=", v)
+    end
+    
+    self.save
+  end
+
+
+  def normalize_attributes
+    self.uuid ||= SecureRandom.uuid
+    building_request if self.extract_statuses.empty?
+  end
+
+end
+
+ExtractRequest::ExtractTypeConst.define_consts