terrapop_models 0.1.0

data/db/migrate/20131002213443_add_ipums_email_sent_at_column.rb

@@ -0,0 +1,8 @@
+class AddIpumsEmailSentAtColumn < ActiveRecord::Migration
+  def up
+    add_column :users, :ipumsi_request_email_sent_at, :datetime
+  end
+
+  def down
+  end
+end

data/db/migrate/20131020154130_remove_index_if_exists.rb

@@ -0,0 +1,9 @@
+class RemoveIndexIfExists < ActiveRecord::Migration
+  def up
+    execute "DROP INDEX IF EXISTS index_ui_text_snippet_on_key_text"
+    execute "DROP TABLE IF EXISTS text_snippet"
+  end
+
+  def down
+  end
+end

data/db/migrate/20131020170642_remove_ui_snippet_table_if_exists.rb

@@ -0,0 +1,8 @@
+class RemoveUiSnippetTableIfExists < ActiveRecord::Migration
+  def up
+    execute "DROP TABLE IF EXISTS text_snippets"
+  end
+
+  def down
+  end
+end

data/db/migrate/20131021204227_create_ui_text_snippet.rb

@@ -0,0 +1,11 @@
+class CreateUiTextSnippet < ActiveRecord::Migration
+  def change
+    create_table :ui_text_snippet do |t|
+      t.column  :key_text,      :text
+      t.column  :text_snippet,  :text
+      t.timestamps
+    end
+
+    add_index :ui_text_snippet, :key_text
+  end
+end

data/db/migrate/20131022190227_rename_table_fix_ui_text_snippets.rb

@@ -0,0 +1,9 @@
+class RenameTableFixUiTextSnippets < ActiveRecord::Migration
+  def up
+    execute "DROP TABLE IF EXISTS ui_text_snippets"
+    rename_table :ui_text_snippet, :ui_text_snippets
+  end
+
+  def down
+  end
+end

data/db/migrate/20131119154817_add_unique_index_on_uuid_on.rb

@@ -0,0 +1,9 @@
+class AddUniqueIndexOnUuidOn < ActiveRecord::Migration
+  def up
+    remove_index :extract_requests, :uuid
+    add_index :extract_requests, :uuid, :unique => true #, :name => :unique_extract_requests_uuid_index
+  end
+
+  def down
+  end
+end

data/db/migrate/20131202161911_add_send_toi_rods_flag_to_extract_request.rb

@@ -0,0 +1,13 @@
+class AddSendToiRodsFlagToExtractRequest < ActiveRecord::Migration
+  def change
+    
+    add_column :extract_requests, :send_to_irods, :boolean, :default => false
+    add_index :extract_requests, :send_to_irods
+  
+    ExtractRequest.all.each{|extract|
+      extract.send_to_irods = false
+      extract.save
+    }
+    
+  end
+end

data/db/migrate/20140122150020_add_geographic_levelto_geog_instance.rb

@@ -0,0 +1,11 @@
+class AddGeographicLeveltoGeogInstance < ActiveRecord::Migration
+  def up
+    add_column :geog_instances, :geog_code, :string, :limit => 10
+    add_column :geog_instances, :str_code, :string, :limit => 16
+    add_index :geog_instances, :geog_code
+    add_index :geog_instances, :str_code
+  end
+
+  def down
+  end
+end

data/db/migrate/20140124184809_add_zip_file_location.rb

@@ -0,0 +1,8 @@
+class AddZipFileLocation < ActiveRecord::Migration
+  def up
+    add_column :extract_requests, :extract_url, :text
+  end
+
+  def down
+  end
+end

data/db/migrate/20140205205539_add_api_related_fields_to_user.rb

@@ -0,0 +1,5 @@
+class AddApiRelatedFieldsToUser < ActiveRecord::Migration
+  def change
+    add_column :users, :api_key, :string, :limit => 256
+  end
+end

data/db/migrate/20140206213258_create_extract_request_error_events.rb

@@ -0,0 +1,14 @@
+class CreateExtractRequestErrorEvents < ActiveRecord::Migration
+  def change
+    
+    create_table :extract_request_error_events do |t|
+      t.column    :extract_request_id, :bigint
+      t.column    :error_event_id, :bigint
+      t.timestamps
+    end
+    
+    foreign_key :extract_request_error_events, :extract_request_id
+    foreign_key :extract_request_error_events, :error_event_id
+    
+  end
+end

data/db/migrate/20140206222630_add_requesting_url_to_extract_request.rb

@@ -0,0 +1,5 @@
+class AddRequestingUrlToExtractRequest < ActiveRecord::Migration
+  def change
+    add_column :extract_requests, :request_url, :string, :limit => 256
+  end
+end

data/db/migrate/20140207153005_create_api_logs.rb

@@ -0,0 +1,14 @@
+class CreateApiLogs < ActiveRecord::Migration
+  def change
+    
+    create_table :api_logs do |t|
+      t.column      :api_key, :text
+      t.column      :action,  :text
+      t.column      :extra,   :text
+      t.timestamps
+    end
+    
+    add_index :api_logs, :api_key
+    
+  end
+end

data/db/migrate/20140228182300_change_user_institutional_affiliation_to_bool.rb

@@ -0,0 +1,50 @@
+class ChangeUserInstitutionalAffiliationToBool < ActiveRecord::Migration
+  class User < ActiveRecord::Base
+  end
+
+  def up
+    # add a temp column for the data transfer
+    add_column :users, :institutional_affiliation_temp, :boolean
+
+    # transfer the data
+    User.reset_column_information
+    User.all.each do |user|
+      if user.institutional_affiliation == 'yes'
+        user.institutional_affiliation_temp = true
+      elsif user.institutional_affiliation == 'no'
+        user.institutional_affiliation_temp = false
+      end
+      user.save
+    end
+
+    # drop the original column
+    remove_column :users, :institutional_affiliation
+
+    # rename the temp column to the original one
+    rename_column :users, :institutional_affiliation_temp, :institutional_affiliation
+
+  end
+
+  def down
+    # add a temp column for the data transfer
+    add_column :users, :institutional_affiliation_temp, :string, :limit => 128
+
+    # transfer the data
+    User.reset_column_information
+    User.all.each do |user|
+      if user.institutional_affiliation == true
+        user.institutional_affiliation_temp = 'yes'
+      elsif user.institutional_affiliation == false
+        user.institutional_affiliation_temp = 'no'
+      end
+      user.save
+    end
+
+    # drop the original column
+    remove_column :users, :institutional_affiliation
+
+    # rename the temp column to the original one
+    rename_column :users, :institutional_affiliation_temp, :institutional_affiliation
+
+  end
+end

data/db/migrate/20140312143201_add_start_and_end_time_and_total_time_to_extract_request.rb

@@ -0,0 +1,7 @@
+class AddStartAndEndTimeAndTotalTimeToExtractRequest < ActiveRecord::Migration
+  def change
+    add_column :extract_requests, :begin_extract_time, :bigint
+    add_column :extract_requests, :finish_extract_time, :bigint
+    add_column :extract_requests, :total_time, :bigint
+  end
+end

data/db/migrate/20140318212913_remove_add_index_to_area_data_value.rb

@@ -0,0 +1,12 @@
+class RemoveAddIndexToAreaDataValue < ActiveRecord::Migration
+  def up
+    
+    remove_column :area_data_values, :mnemonic
+    add_column :area_data_values, :mnemonic, :string, :limit => 16
+    add_index :area_data_values, :mnemonic
+    
+  end
+
+  def down
+  end
+end

data/db/migrate/20140319155819_add_long_mnemonic_to_area_data_value.rb

@@ -0,0 +1,7 @@
+class AddLongMnemonicToAreaDataValue < ActiveRecord::Migration
+  def change
+    remove_column :area_data_values, :mnemonic
+    add_column :area_data_values, :mnemonic, :string, :limit => 32
+    add_index :area_data_values, :mnemonic
+  end
+end

data/db/migrate/20140319200718_drop_mnemonic_column_area_data_value.rb

@@ -0,0 +1,10 @@
+class DropMnemonicColumnAreaDataValue < ActiveRecord::Migration
+  def up
+    remove_column :area_data_values, :mnemonic
+    add_column :area_data_values, :synthetic_mnemonic, :string, :limit => 32
+    add_index :area_data_values, :synthetic_mnemonic
+  end
+
+  def down
+  end
+end

data/db/migrate/20140320184210_add_show_in_ui_column_in_raster_variable.rb

@@ -0,0 +1,8 @@
+class AddShowInUiColumnInRasterVariable < ActiveRecord::Migration
+  def up
+    add_column :raster_variables, :show_in_ui, :boolean, :default => true
+  end
+
+  def down
+  end
+end

data/db/migrate/20140328150435_rename_revision_of_column_to_rabl_usable_column_name.rb

@@ -0,0 +1,8 @@
+class RenameRevisionOfColumnToRablUsableColumnName < ActiveRecord::Migration
+  def up
+    rename_column :extract_requests, :revision_of, :revision_of_id
+  end
+
+  def down
+  end
+end

data/db/migrate/20140331142153_add_revision_of_column.rb

@@ -0,0 +1,8 @@
+class AddRevisionOfColumn < ActiveRecord::Migration
+  def up
+    add_column :extract_requests, :revision_of, :bigint
+  end
+
+  def down
+  end
+end

data/db/migrate/20140402181324_add_new_raster_summary_functions.rb

@@ -0,0 +1,191 @@
+class AddNewRasterSummaryFunctions < ActiveRecord::Migration
+  # Note that the PostgreSQL JDBC driver has problems creating stored proces that involve the DECLARE keyword.
+  def self.up
+    # this version of the stored proc isn't usable because it creates a local variable via the DECLARE keyword, which breaks the PostgreSQL 9.1.4 JDBC driver.
+    summary_calc_sql =<<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_raster_summary_calc_v2(raster_op_name varchar(32), rast raster)
+  RETURNS numeric(20,4) as $$
+DECLARE
+  summary_val numeric(20,4);
+BEGIN
+  CASE raster_op_name
+    WHEN 'max' THEN summary_val := CAST((ST_SummaryStats(rast)).max as numeric(20,4));
+    WHEN 'min' THEN summary_val := CAST((ST_SummaryStats(rast)).min as numeric(20,4));
+    WHEN 'mean' THEN summary_val := CAST((ST_SummaryStats(rast)).mean as numeric(20,4));
+    WHEN 'count' THEN summary_val := CAST((ST_SummaryStats(rast)).count as numeric(20,4));
+    WHEN 'sum' THEN summary_val := CAST((ST_SummaryStats(rast)).sum as numeric(20,4));
+    WHEN 'mode' THEN SELECT histogram.value into summary_val
+                      FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram
+                      ORDER BY histogram.count desc limit 1;
+    ELSE summary_val := null;
+  END CASE;
+
+  RETURN summary_val;
+END;
+$$ LANGUAGE plpgsql
+END_OF_PROC
+
+    # so, since declare is off-limits, we can use a different structure which requires having a utility function.
+    # This function is also somewhat awkward, the RETURN at the end returns the value in the single OUT parameter
+    # as declared in the RETURNS part of the signature.
+
+    # Note that we can't use SELECT INTO STRICT here because the return value might be NULL, if the whole raster tile is nodata.
+
+    summary_calc_modal_sql = <<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_modal_value_v2(rast raster, OUT modal numeric) RETURNS numeric
+  LANGUAGE plpgsql
+  AS $$
+BEGIN
+  SELECT histogram.value into modal
+    FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram
+    ORDER BY histogram.count desc limit 1;
+  return;
+END;
+$$;
+END_OF_PROC
+
+    summary_calc_num_classes_sql = <<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_num_classes_v2(rast raster, OUT num_classes numeric) RETURNS numeric
+  LANGUAGE plpgsql
+  AS $$
+BEGIN
+  SELECT count(histogram.*) into num_classes
+    FROM (SELECT (ST_ValueCount(rast,1)).*) As histogram;
+  return;
+END;
+$$;
+END_OF_PROC
+
+
+    summary_calc_nodeclare_sql =<<-END_OF_PROC
+CREATE OR REPLACE FUNCTION terrapop_raster_summary_calc_v2(raster_op_name varchar(32), rast raster, area double precision) RETURNS numeric
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+  RETURN CASE raster_op_name
+    WHEN 'max' THEN CAST((ST_SummaryStats(rast)).max as double precision)
+    WHEN 'min' THEN CAST((ST_SummaryStats(rast)).min as double precision)
+    WHEN 'mean' THEN CAST((ST_SummaryStats(rast)).mean as double precision)
+    WHEN 'count' THEN CAST((ST_SummaryStats(rast)).count as double precision)
+    WHEN 'sum' THEN CAST((ST_SummaryStats(rast)).sum as double precision)
+    WHEN 'mode' THEN terrapop_modal_value_v2(rast)
+    WHEN 'num_classes' THEN terrapop_num_classes_v2(rast)
+    WHEN 'total_area_bin' THEN CAST((ST_SummaryStats(rast)).sum as double precision)
+    WHEN 'percent_area_bin' THEN CAST(((ST_SummaryStats(rast)).sum  / area ) as double precision)
+    WHEN 'total_area_areal' THEN CAST((ST_SummaryStats(rast)).sum as double precision)
+    WHEN 'percent_area_areal' THEN CAST(((ST_SummaryStats(rast)).sum  / area ) as double precision)
+    ELSE null
+  END;
+END;
+$$;
+END_OF_PROC
+
+    summary_sql = <<-END_OF_PROC
+    CREATE OR REPLACE FUNCTION terrapop_raster_summary_v3(sample_geog_lvl_id bigint, raster_var_id bigint, raster_var_ref_id bigint, raster_op_name varchar(32))
+    RETURNS TABLE(sample_geog_level_id bigint, raster_variable_id bigint, raster_operation_name varchar(32), geog_instance_id bigint,
+      geog_instance_label varchar(255), geog_instance_code numeric(20,0), raster_mnemonic varchar(255),
+      boundary_area double precision, raster_area double precision, summary_value double precision) AS $$
+    BEGIN
+        RETURN QUERY
+        
+           SELECT final.sample_geog_level_id, final.raster_variable_id, final.raster_op_name, final.geog_instance_id,
+                  final.geog_instance_label, final.geog_instance_code, final.mnemonic, final.boundary_area, final.raster_area, final.value FROM (
+              select unioned_rast.sample_geog_level_id, unioned_rast.raster_variable_id, raster_op_name,
+                     unioned_rast.geog_instance_id, unioned_rast.geog_instance_label, unioned_rast.geog_instance_code,
+                     CAST(unioned_rast.raster_variable_name || '_' || raster_op_name as varchar(255)) as mnemonic,
+                     unioned_rast.boundary_area,
+                     unioned_rast.raster_area,
+                     terrapop_raster_summary_calc_v2(raster_op_name, St_Union(unioned_rast.rast), unioned_rast.raster_area)::double precision as value
+              from (
+                select base.sample_geog_level_id, base.raster_variable_id, base.geog_instance_id, base.geog_instance_label,
+                      base.geog_instance_code, base.raster_variable_name,
+                      base.boundary_area::double precision AS boundary_area,
+                      /*ST_Union(ST_MapAlgebra(base.rast, ref_base.rast, '([rast1] * [rast2])::float')) as rast,*/
+                      /*ST_GeoReference(ST_Union(base.rast), 'GDAL') AS base_esri_ref,*/
+                      ST_Union(ST_MapAlgebra(base.rast, ST_SetGeoReference(ref_base.rast, ST_GeoReference(base.rast, 'GDAL') , 'GDAL'), '([rast1] * [rast2])::float')) as rast ,
+                      terrapop_raster_area_v1(base.geog_instance_id, raster_var_ref_id)::double precision AS raster_area
+                from (
+                  SELECT sgl.id as "sample_geog_level_id", my_raster.raster_variable_id as "raster_variable_id",
+                    gi.id as "geog_instance_id", gi.label as "geog_instance_label", gi.code as "geog_instance_code",
+                    my_raster.name as "raster_variable_name", ST_AREA(bound.geog) as boundary_area,
+                    ST_Union(ST_Clip(my_raster.rast, bound.geog::geometry)) as rast
+                  FROM sample_geog_levels sgl
+                  inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                  inner join boundaries bound on bound.geog_instance_id = gi.id
+                  inner join rasters my_raster on ST_Intersects(my_raster.rast, bound.geog::geometry)
+                  where sgl.id = sample_geog_lvl_id and my_raster.raster_variable_id = raster_var_id
+                  group by sgl.id, my_raster.raster_variable_id, gi.id, gi.label, gi.code, my_raster.name, boundary_area
+                ) base,
+                ( SELECT ST_Union(ST_Clip(my_raster.rast, bound.geog::geometry)) as rast
+                          FROM sample_geog_levels sgl
+                          inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                          inner join boundaries bound on bound.geog_instance_id = gi.id
+                          inner join rasters my_raster on ST_Intersects(my_raster.rast, bound.geog::geometry)
+                          where sgl.id = sample_geog_lvl_id and my_raster.raster_variable_id = raster_var_ref_id
+                ) ref_base
+                group by base.sample_geog_level_id, base.raster_variable_id, base.geog_instance_label,
+                    base.geog_instance_id, base.geog_instance_code, base.raster_variable_name,
+                    base.boundary_area
+              ) unioned_rast
+              group by
+                unioned_rast.sample_geog_level_id, unioned_rast.raster_variable_id,
+                raster_op_name, unioned_rast.geog_instance_id, unioned_rast.geog_instance_label,
+                unioned_rast.geog_instance_code, mnemonic, unioned_rast.boundary_area, unioned_rast.raster_area
+              order by unioned_rast.geog_instance_code
+            ) final
+            group by final.sample_geog_level_id,
+              final.raster_variable_id,
+              final.raster_op_name,
+              final.geog_instance_id,
+              final.geog_instance_label,
+              final.geog_instance_code,
+              final.mnemonic,
+              final.boundary_area,
+              final.raster_area,
+              final.value
+            order by final.geog_instance_code;
+
+    END;
+    $$ LANGUAGE plpgsql
+END_OF_PROC
+          
+
+    area_sql = <<-END_OF_PROC
+    CREATE OR REPLACE FUNCTION terrapop_raster_area_v1(_geog_instance_id bigint, raster_var_ref_id bigint)
+    RETURNS TABLE(raster_area double precision) AS $$
+    BEGIN
+        RETURN QUERY
+        SELECT final.raster_area FROM (
+            select unioned_rast.raster_area AS raster_area
+            from (
+              select SUM((ST_SummaryStats((base.rast))).sum::double precision) AS raster_area
+              from (
+                SELECT ST_Union(ST_Clip(my_raster.rast, bound.geog::geometry)) as rast
+                FROM geog_instances gi, boundaries bound
+                inner join rasters my_raster on ST_Intersects(my_raster.rast, bound.geog::geometry)
+                where gi.id = _geog_instance_id and my_raster.raster_variable_id = raster_var_ref_id AND bound.geog_instance_id = gi.id
+                group by my_raster.raster_variable_id, gi.id, gi.label, gi.code, my_raster.name
+              ) base
+            ) unioned_rast
+          ) final
+          group by final.raster_area;
+    END;
+    $$ LANGUAGE plpgsql
+END_OF_PROC
+
+    execute summary_calc_modal_sql
+    execute summary_calc_num_classes_sql
+    execute summary_calc_nodeclare_sql
+    execute area_sql
+    execute summary_sql
+  end
+
+  def self.down
+    execute 'drop function if exists terrapop_modal_value_v2(rast raster, OUT modal numeric)'
+    execute 'drop function if exists terrapop_num_classes_v2(rast raster, OUT num_classes numeric)'
+    execute 'drop function if exists terrapop_raster_summary_calc_v2(raster_op_name varchar(32), rast raster, area float)'
+    execure 'drop function if exists terrapop_raster_area_v1(sample_geog_lvl_id bigint, raster_var_ref_id bigint)'
+    execute 'drop function if exists terrapop_raster_summary_v3(sample_geog_lvl_id bigint, raster_var_id bigint, raster_var_ref_id bigint, raster_op_name varchar(32))'
+    
+  end
+end