terrapop_models 0.1.0

data/db/migrate/00000000010300_create_extract_requests_labels_join_table.rb

@@ -0,0 +1,20 @@
+class CreateExtractRequestsLabelsJoinTable < ActiveRecord::Migration
+  def up
+
+    create_table :extract_requests_labels, :id => false do |t|
+      t.column      :extract_request_id, :bigint
+      t.column      :label_id,           :bigint
+      t.column      :user_id,            :bigint
+      t.column      :visible,            :boolean, :default => true, :null => false
+    end
+
+    foreign_key(:extract_requests_labels, :extract_request_id)
+    foreign_key_raw(:extract_requests_labels, :label_id, :tags, :id)
+    foreign_key(:extract_requests_labels, :user_id)
+
+  end
+
+  def down
+    drop table :extract_requests_labels
+  end
+end

data/db/migrate/00000000010400_create_extract_statuses.rb

@@ -0,0 +1,13 @@
+class CreateExtractStatuses < ActiveRecord::Migration
+  def change
+    create_table :extract_statuses do |t|
+      t.column      :extract_request_id,          :bigint
+      t.column      :status,                      :string,  :limit => 128,  :default=>'building request'
+      t.column      :status_definition_id,        :bigint
+      t.timestamps
+    end
+
+    foreign_key(:extract_statuses, :extract_request_id)
+
+  end
+end

data/db/migrate/00000000010500_create_request_geog_units.rb

@@ -0,0 +1,13 @@
+class CreateRequestGeogUnits < ActiveRecord::Migration
+  def change
+    create_table :request_geog_units do |t|
+      t.column :extract_request_id, :bigint,  :null=>false
+      t.column :geog_unit_id,       :bigint,  :null=>false
+
+      t.timestamps
+    end
+
+    foreign_key(:request_geog_units, :extract_request_id)
+    foreign_key(:request_geog_units, :geog_unit_id)
+  end
+end

data/db/migrate/00000000010600_create_request_terrapop_samples.rb

@@ -0,0 +1,15 @@
+class CreateRequestTerrapopSamples < ActiveRecord::Migration
+  def change
+    create_table :request_terrapop_samples do |t|
+      
+      t.column :terrapop_sample_id, :bigint, :null=>false
+      t.column :extract_request_id, :bigint, :null=>false
+      t.timestamps
+      
+    end
+
+    foreign_key(:request_terrapop_samples, :terrapop_sample_id)
+    foreign_key(:request_terrapop_samples, :extract_request_id)
+
+  end
+end

data/db/migrate/00000000011000_create_request_variable.rb

@@ -0,0 +1,16 @@
+class CreateRequestVariable < ActiveRecord::Migration
+   def  change
+    create_table :request_variables do |t|
+      t.column :variable_id,          :bigint, :null=>false
+      t.column :extract_request_id,   :bigint, :null=>false      
+      t.timestamps
+      
+    end
+
+    foreign_key(:request_variables, :variable_id)
+    foreign_key(:request_variables, :extract_request_id)
+
+    add_index :request_variables,:extract_request_id
+    add_index :request_variables,:variable_id
+  end
+end

data/db/migrate/00000000011100_create_request_area_data_variables.rb

@@ -0,0 +1,18 @@
+class CreateRequestAreaDataVariables < ActiveRecord::Migration
+
+  def change
+  
+    create_table :request_area_data_variables do |t|
+      t.column :area_data_variable_id,   :bigint, :null=>false
+      t.column :extract_request_id,      :bigint, :null=>false
+      t.timestamps  
+    end # create table
+  
+    foreign_key(:request_area_data_variables, :extract_request_id)
+
+    add_index :request_area_data_variables, :area_data_variable_id
+    add_index :request_area_data_variables, :extract_request_id
+  
+  end # change
+  
+end

data/db/migrate/00000000011200_create_request_raster_variables.rb

@@ -0,0 +1,20 @@
+class CreateRequestRasterVariables < ActiveRecord::Migration
+  def change
+  
+    create_table :request_raster_variables do |t|
+      t.column  :raster_variable_id,    :bigint, :null=>false
+      t.column  :extract_request_id,    :bigint, :null=>false
+      t.column  :raster_operation_id,   :bigint, :null=>true     # this can be null for raster extracts, but should be populated for area_level extracts
+      t.timestamps  
+    end # create table
+
+    foreign_key(:request_raster_variables, :raster_variable_id)
+    foreign_key(:request_raster_variables, :extract_request_id)
+    
+    add_index :request_raster_variables, :raster_variable_id
+    add_index :request_raster_variables, :extract_request_id
+    add_index :request_raster_variables, :raster_operation_id
+  
+  end # change
+  
+end

data/db/migrate/00000000012000_create_raster_summary_functions.rb

@@ -0,0 +1,130 @@
+class CreateRasterSummaryFunctions < ActiveRecord::Migration
+
+  # Note that the PostgreSQL JDBC driver has problems creating stored proces that involve the DECLARE keyword.
+  def self.up
+    # this version of the stored proc isn't usable because it creates a local variable via the DECLARE keyword, which breaks the PostgreSQL 9.1.4 JDBC driver.
+    summary_calc_sql =<<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_raster_summary_calc(raster_op_name varchar(32), rast raster)
+  RETURNS numeric(20,4) as $$
+DECLARE
+  summary_val numeric(20,4);
+BEGIN
+  CASE raster_op_name
+    WHEN 'max' THEN summary_val := CAST((ST_SummaryStats(rast)).max as numeric(20,4));
+    WHEN 'min' THEN summary_val := CAST((ST_SummaryStats(rast)).min as numeric(20,4));
+    WHEN 'mean' THEN summary_val := CAST((ST_SummaryStats(rast)).mean as numeric(20,4));
+    WHEN 'count' THEN summary_val := CAST((ST_SummaryStats(rast)).count as numeric(20,4));
+    WHEN 'sum' THEN summary_val := CAST((ST_SummaryStats(rast)).sum as numeric(20,4));
+    WHEN 'mode' THEN SELECT histogram.value into summary_val
+                      FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram
+                      ORDER BY histogram.count desc limit 1;
+    ELSE summary_val := null;
+  END CASE;
+
+  RETURN summary_val;
+END;
+$$ LANGUAGE plpgsql
+END_OF_PROC
+
+    # so, since declare is off-limits, we can use a different structure which requires having a utility function.
+    # This function is also somewhat awkward, the RETURN at the end returns the value in the single OUT parameter
+    # as declared in the RETURNS part of the signature.
+
+    # Note that we can't use SELECT INTO STRICT here because the return value might be NULL, if the whole raster tile is nodata.
+
+    summary_calc_modal_sql = <<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_modal_value(rast raster, OUT modal numeric) RETURNS numeric
+  LANGUAGE plpgsql
+  AS $$
+BEGIN
+  SELECT histogram.value into modal
+    FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram
+    ORDER BY histogram.count desc limit 1;
+  return;
+END;
+$$;
+END_OF_PROC
+
+    summary_calc_num_classes_sql = <<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_num_classes(rast raster, OUT num_classes numeric) RETURNS numeric
+  LANGUAGE plpgsql
+  AS $$
+BEGIN
+  SELECT count(histogram.*) into num_classes
+    FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram;
+  return;
+END;
+$$;
+END_OF_PROC
+
+
+    summary_calc_nodeclare_sql =<<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_raster_summary_calc(raster_op_name varchar(32), rast raster, area float) RETURNS numeric
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+  RETURN CASE raster_op_name
+    WHEN 'max' THEN CAST((ST_SummaryStats(rast)).max as numeric(20,4))
+    WHEN 'min' THEN CAST((ST_SummaryStats(rast)).min as numeric(20,4))
+    WHEN 'mean' THEN CAST((ST_SummaryStats(rast)).mean as numeric(20,4))
+    WHEN 'count' THEN CAST((ST_SummaryStats(rast)).count as numeric(20,4))
+    WHEN 'sum' THEN CAST((ST_SummaryStats(rast)).sum as numeric(20,4))
+    WHEN 'mode' THEN terrapop_modal_value(rast)
+    WHEN 'num_classes' THEN terrapop_num_classes(rast)
+    WHEN 'total_area_areal' THEN CAST((ST_SummaryStats(rast)).sum as numeric(20,4))
+    WHEN 'percent_area_areal' THEN CAST(((ST_SummaryStats(rast)).sum  / area ) as numeric(20,4))
+    ELSE null
+  END;
+END;
+$$;
+END_OF_PROC
+
+    summary_sql = <<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_raster_summary(sample_geog_lvl_id bigint, raster_var_id bigint, raster_op_name varchar(32))
+RETURNS TABLE(sample_geog_level_id bigint, raster_variable_id bigint, raster_operation_name varchar(32), geog_instance_id bigint,
+  geog_instance_label varchar(255), geog_instance_code numeric(20,0), raster_mnemonic varchar(255),
+  boundary_area double precision, summary_value numeric(20,4)) AS $$
+BEGIN
+    RETURN QUERY
+      select unioned_rast.sample_geog_level_id, unioned_rast.raster_variable_id, raster_op_name,
+             unioned_rast.geog_instance_id, unioned_rast.geog_instance_label, unioned_rast.geog_instance_code,
+             CAST(unioned_rast.raster_variable_name || '_' || raster_op_name as varchar(255)) as mnemonic,
+             unioned_rast.boundary_area,
+             terrapop_raster_summary_calc(raster_op_name, unioned_rast.rast, unioned_rast.boundary_area) as value
+      from (
+        select base.sample_geog_level_id, base.raster_variable_id, base.geog_instance_id, base.geog_instance_label,
+              base.geog_instance_code, base.raster_variable_name, base.boundary_area,
+              st_union(base.rast) as rast
+        from (
+          SELECT sgl.id as "sample_geog_level_id", my_raster.raster_variable_id as "raster_variable_id",
+            gi.id as "geog_instance_id", gi.label as "geog_instance_label", gi.code as "geog_instance_code",
+            my_raster.name as "raster_variable_name", ST_AREA(bound.geog) as boundary_area,
+            ST_Clip(my_raster.rast, bound.geog::geometry) as rast
+          FROM sample_geog_levels sgl
+          inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+          inner join boundaries bound on bound.geog_instance_id = gi.id
+          inner join rasters my_raster on ST_Intersects(my_raster.rast, bound.geog::geometry)
+          where sgl.id = sample_geog_lvl_id and my_raster.raster_variable_id = raster_var_id
+        ) base
+        group by base.sample_geog_level_id, base.raster_variable_id, base.geog_instance_label,
+            base.geog_instance_id, base.geog_instance_code, base.raster_variable_name,
+            base.boundary_area
+      ) unioned_rast
+      order by unioned_rast.geog_instance_code;
+END;
+$$ LANGUAGE plpgsql
+END_OF_PROC
+
+    execute summary_calc_modal_sql
+    execute summary_calc_num_classes_sql
+    execute summary_calc_nodeclare_sql
+    execute summary_sql
+  end
+
+  def self.down
+    execute 'drop function if exists terrapop_modal_value(rast raster, OUT modal numeric)'
+    execute 'drop function if exists terrapop_num_classes(rast raster, OUT num_classes numeric)'
+    execute 'drop function if exists terrapop_raster_summary_calc(raster_op_name varchar(32), rast raster)'
+    execute 'drop function if exists terrapop_raster_summary(sample_geog_lvl_id bigint, raster_var_id bigint, raster_op_name varchar(32))'
+  end
+end

data/db/migrate/00000000013000_create_terrapop_settings.rb

@@ -0,0 +1,24 @@
+class CreateTerrapopSettings < ActiveRecord::Migration
+
+  def change
+
+    create_table :terrapop_settings do |t|
+      t.hstore :data
+      t.timestamps
+    end
+
+    ###
+    #
+    # Using a GIN index instead of a GIST index
+    #
+    # GIN indexes take longer to create but are generally faster to query
+    #
+    # more information:
+    # http://www.postgresql.org/docs/9.2/static/textsearch-indexes.html
+    #
+
+    execute "CREATE INDEX terrapop_settings_gin_data ON terrapop_settings USING GIN(data)"
+
+  end
+
+end

data/db/migrate/20121218190647_create_attached_variable_pointer.rb

@@ -0,0 +1,18 @@
+class CreateAttachedVariablePointer < ActiveRecord::Migration
+  def up
+    
+   # Basically a list of valid pointers. The mnemonic matches mnemonics of microdata variables in the 'variables' table.
+   # We don't need a foreign key or anything; these are mostly for selection and testing purposes.
+    create_table "attached_variable_pointers", :force => true do |t|
+      t.string "mnemonic"
+      t.string "suffix"
+      t.string "label"
+      t.timestamps
+    end
+    
+  end
+
+  def down
+    #drop_table :attached_variable_pointers
+  end
+end

data/db/migrate/20121218191609_add_attached_variable_pointer_id.rb

@@ -0,0 +1,9 @@
+class AddAttachedVariablePointerId < ActiveRecord::Migration
+  def up
+    add_column :request_variables,:attached_variable_pointer_id, :integer
+  end
+
+  def down
+    remove_column :request_variables, :attached_variable_pointer_id
+  end
+end

data/db/migrate/20121218205322_add_wants_attached.rb

@@ -0,0 +1,9 @@
+class AddWantsAttached < ActiveRecord::Migration
+  def up
+    add_column :request_variables,:wants_attached,:boolean
+  end
+
+  def down
+    remove_column :request_variables,:wants_attached,:boolean
+  end
+end

data/db/migrate/20121219145142_add_attributes_to_terrapop_sample.rb

@@ -0,0 +1,68 @@
+class AddAttributesToTerrapopSample < ActiveRecord::Migration
+  def change
+
+    # Census characteristics
+    add_column :terrapop_samples, :local_title,             :text
+    add_column :terrapop_samples, :census_agency,           :text
+    add_column :terrapop_samples, :population_universe,     :text
+    add_column :terrapop_samples, :de_jure_or_de_facto,     :text   # ? Can be either or both (see brazil 1960)
+    add_column :terrapop_samples, :enumeration_unit,        :text   # (could maybe abstract to a list, not sure of all possible types. Dwelling, 
+    add_column :terrapop_samples, :census_start_date,       :date
+    add_column :terrapop_samples, :census_end_date,         :date
+    add_column :terrapop_samples, :fieldwork_start_period,  :date   # ? have start and end dates?
+    add_column :terrapop_samples, :fieldwork_end_period,    :date
+    add_column :terrapop_samples, :enumeration_forms,       :text
+    add_column :terrapop_samples, :type_of_fieldwork,       :text
+    add_column :terrapop_samples, :respondent,              :text
+    add_column :terrapop_samples, :coverage,                :float    # with null - null indicates "no official estimate"?
+    add_column :terrapop_samples, :undercount,              :float    # : same as coverage
+    add_column :terrapop_samples, :undercount_notes,        :text
+    
+    # Microdata sample characteristics
+    add_column :terrapop_samples, :microdata_source,              :text
+    add_column :terrapop_samples, :long_form_sample_design,       :text
+    add_column :terrapop_samples, :mpc_sample_design,             :text
+    add_column :terrapop_samples, :sample_unit,                   :text
+    add_column :terrapop_samples, :sample_fraction,               :float
+    add_column :terrapop_samples, :sample_fraction_notes,         :text
+    add_column :terrapop_samples, :sample_size,                   :bigint
+    add_column :terrapop_samples, :sample_weights,                :text
+    add_column :terrapop_samples, :sample_characteristics_notes,  :text
+    add_column :terrapop_samples, :sample_general_notes,          :text
+
+    # Units identified
+    add_column :terrapop_samples, :has_dwellings,           :boolean
+    add_column :terrapop_samples, :has_dwellings_note,      :text
+    add_column :terrapop_samples, :has_vacant_units,        :boolean
+    add_column :terrapop_samples, :has_vacant_units_note,   :text
+    add_column :terrapop_samples, :has_closed_units,        :boolean
+    add_column :terrapop_samples, :has_closed_units_note,   :text
+    add_column :terrapop_samples, :smallest_geography,      :text
+    add_column :terrapop_samples, :has_households,                  :boolean
+    add_column :terrapop_samples, :has_households_note,             :text
+    add_column :terrapop_samples, :has_families,                    :boolean
+    add_column :terrapop_samples, :has_families_note,               :text
+    add_column :terrapop_samples, :has_individuals,                 :boolean
+    add_column :terrapop_samples, :has_individuals_note,            :text
+    add_column :terrapop_samples, :has_group_quarters,              :boolean
+    add_column :terrapop_samples, :has_group_quarters_note,         :text
+    add_column :terrapop_samples, :has_indigenous_pop,              :boolean
+    add_column :terrapop_samples, :has_indigenous_pop_note,         :text
+    add_column :terrapop_samples, :has_special_pop,                 :boolean
+    add_column :terrapop_samples, :has_special_pop_note,            :text
+    add_column :terrapop_samples, :units_notes,                     :text
+    add_column :terrapop_samples, :identification_general_notes,    :text
+    
+    # Unit definitions
+    add_column :terrapop_samples, :unit_definition_household,                 :text
+    add_column :terrapop_samples, :unit_definition_family,                    :text
+    add_column :terrapop_samples, :unit_definition_dwelling,                  :text
+    add_column :terrapop_samples, :unit_definition_group_quarters,            :text
+    add_column :terrapop_samples, :unit_definition_homeless_population,       :text
+    add_column :terrapop_samples, :unit_definition_institution,               :text
+    add_column :terrapop_samples, :unit_definition_institutional_population,  :text
+    add_column :terrapop_samples, :unit_definition_notes,                     :text
+    add_column :terrapop_samples, :unit_definition_general_notes,             :text
+
+  end
+end

data/db/migrate/20130125155216_add_default_order_index_to_variables.rb

@@ -0,0 +1,5 @@
+class AddDefaultOrderIndexToVariables < ActiveRecord::Migration
+  def change
+    add_index :variables, :default_order
+  end
+end

data/db/migrate/20130129204515_create_request_samples.rb

@@ -0,0 +1,14 @@
+class CreateRequestSamples < ActiveRecord::Migration
+  def change
+    
+    create_table :request_samples do |t|
+      t.column :sample_id,          :bigint, :null => false
+      t.column :extract_request_id, :bigint, :null => false
+      t.timestamps
+    end
+    
+    foreign_key(:request_samples, :sample_id)
+    foreign_key(:request_samples, :extract_request_id)
+    
+  end
+end

data/db/migrate/20130129204718_create_request_raster_datasets.rb

@@ -0,0 +1,13 @@
+class CreateRequestRasterDatasets < ActiveRecord::Migration
+  def change
+    create_table :request_raster_datasets do |t|
+      t.column :raster_dataset_id,  :bigint, :null => false
+      t.column :extract_request_id, :bigint, :null => false
+      t.timestamps
+    end
+    
+    foreign_key(:request_raster_datasets, :raster_dataset_id)
+    foreign_key(:request_raster_datasets, :extract_request_id)
+    
+  end
+end

data/db/migrate/20130205195304_create_error_events.rb

@@ -0,0 +1,10 @@
+class CreateErrorEvents < ActiveRecord::Migration
+  def change
+    create_table :error_events do |t|
+      t.column    :user_id,         :bigint,  :default => nil
+      t.column    :message,         :text,    :null => false
+      t.column    :supplementary,   :text
+      t.timestamps
+    end
+  end
+end