terrapop_models 0.1.0

data/db/migrate/20140902194529_add_raster_band_to_raster_dataset_model.rb

@@ -0,0 +1,7 @@
+class AddRasterBandToRasterDatasetModel < ActiveRecord::Migration
+  def change
+    add_column     :raster_datasets, :raster_band_index, :bigint, default: -1, null: false
+    remove_column  :raster_categories, :raster_band_index
+    add_index      :raster_datasets, :raster_band_index
+  end
+end

data/db/migrate/20140903202609_create_heartbeats.rb

@@ -0,0 +1,9 @@
+class CreateHeartbeats < ActiveRecord::Migration
+  def change
+    create_table :heartbeats do |t|
+      t.column :uuid, :string, limit: 36, null: false
+      t.timestamps
+    end
+    add_index :heartbeats, :uuid
+  end
+end

data/db/migrate/20140903204113_create_heartbeat_pulse.rb

@@ -0,0 +1,10 @@
+class CreateHeartbeatPulse < ActiveRecord::Migration
+  def change
+    create_table :heartbeat_pulses do |t|
+      t.column :heartbeat_id, :bigint
+      t.timestamps
+    end
+    foreign_key :heartbeat_pulses, :heartbeat_id
+    add_index :heartbeat_pulses, :heartbeat_id
+  end
+end

data/db/migrate/20140904164247_create_raster_raster_variables.rb

@@ -0,0 +1,16 @@
+class CreateRasterRasterVariables < ActiveRecord::Migration
+  def change
+    create_table :raster_raster_variables do |t|
+      t.column    :raster_id,  :bigint, null: false
+      t.column    :raster_variable_id, :bigint, null: false
+      t.timestamps
+    end
+    
+    add_index :raster_raster_variables, :raster_id
+    add_index :raster_raster_variables, :raster_variable_id
+    
+    foreign_key :raster_raster_variables, :raster_id
+    foreign_key :raster_raster_variables, :raster_variable_id
+    
+  end
+end

data/db/migrate/20140908193007_add_area_level_rasterization_functions.rb

@@ -0,0 +1,103 @@
+class AddAreaLevelRasterizationFunctions < ActiveRecord::Migration
+  def change
+    
+    cell_count_sql = <<-SQL_O_MATIC
+    CREATE OR REPLACE Function terrapop_raster_cellcount(sample_geog_lvl_id bigint, reference_raster_variable_id bigint, raster_sizer bigint)
+        Returns boolean as
+            $BODY$
+            DECLARE
+                zero_count_geom integer;
+            BEGIN
+                WITH new_rast as
+                (
+                SELECT ST_Rescale(ST_Asraster(mybound.geom, my_rast.rast, '32BF', 1.0, -5), my_rast.scale_x/raster_sizer,
+                    my_rast.scale_y/raster_sizer) as rast
+                FROM     (
+                    select ST_Collect(bound.geog::geometry) as geom
+                    from sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                    inner join boundaries bound on bound.geog_instance_id = gi.id
+                    where gi.sample_geog_level_id = sample_geog_lvl_id
+                    ) mybound,
+                    (
+                    select ST_Scalex(r.rast) as scale_x, ST_Scaley(r.rast) as scale_y, r.rast
+                    from rasters r INNER JOIN raster_variables rv ON rv.id = r.raster_variable_id
+                    WHERE rv.id = reference_raster_variable_id
+                    limit 1
+                    ) my_rast
+                )
+                SELECT count(zero_geog.cell_count) into zero_count_geom
+                FROM    (
+                    select gi.id, gi.label,(ST_summarystats(ST_Clip(r.rast, bound.geog::geometry, True),True)).count as cell_count
+                    from sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                    inner join boundaries bound on bound.geog_instance_id = gi.id
+                    inner join new_rast r on ST_Intersects(r.rast,bound.geog::geometry)
+                    where gi.sample_geog_level_id = sample_geog_lvl_id and (ST_summarystats(ST_Clip(r.rast, bound.geog::geometry, True),True)).count = 0
+                    order by gi.id
+                    ) zero_geog;
+
+                IF zero_count_geom > 0 THEN
+                    RETURN True;
+                ELSE
+                    RETURN FALSE;
+                END IF;
+            END;
+            $BODY$
+    LANGUAGE plpgsql;
+    SQL_O_MATIC
+    
+    rasterization_sql = <<-SQL_O_MATIC
+    CREATE OR REPLACE Function terrapop_areal_rasterization(sample_geog_lvl_id bigint, rasters_id bigint, area_data_variables_id bigint, raster_sizer bigint, bnd_num integer)
+        RETURNS TABLE (rast raster)
+            AS
+            $BODY$
+            BEGIN
+                RETURN QUERY
+                    WITH new_rast as
+                    (
+                    SELECT ST_Rescale(ST_Asraster(mybound.geom, my_rast.rast, '32BF', 1.0, -5), my_rast.scale_x/raster_sizer,
+                        my_rast.scale_y/raster_sizer) as rast
+                    FROM     (
+                        select ST_Collect(bound.geog::geometry) as geom
+                        from sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                        inner join boundaries bound on bound.geog_instance_id = gi.id
+                        where gi.sample_geog_level_id = sample_geog_lvl_id
+                        ) mybound,
+                        (
+                        select ST_Scalex(r.rast) as scale_x, ST_Scaley(r.rast) as scale_y, r.rast
+                        from rasters r
+                        where r.raster_variable_id = rasters_id
+                        limit 1
+                        ) my_rast
+                    )
+                   
+                    SELECT ST_Union(ST_AsRaster(clip.geom, new_rast.rast, '32BF', (areal_data.value/clip.cell_count))) as rast
+                    FROM new_rast,
+                      (
+                        select gi.id, adv.mnemonic, area_data.value
+                        from sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                        inner join boundaries bound on bound.geog_instance_id = gi.id
+                        left join area_data_values area_data on gi.id = area_data.geog_instance_id
+                        inner join area_data_variables adv ON area_data.area_data_variable_id = adv.id
+                        where gi.sample_geog_level_id = sample_geog_lvl_id and adv.id = area_data_variables_id
+                      ) areal_data inner join
+                      (
+                        SELECT gi.id, gi.label, Sum(ST_Count(ST_Clip(r.rast, bound.geog::geometry))) as cell_count, ST_union(bound.geog::geometry) as geom
+                        from sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+                        inner join boundaries bound on bound.geog_instance_id = gi.id
+                        inner join new_rast r on ST_Intersects(r.rast,bound.geog::geometry)
+                        where gi.sample_geog_level_id = sample_geog_lvl_id
+                        group by gi.id
+                      ) clip on areal_data.id = clip.id;
+            END;
+            $BODY$
+            LANGUAGE 'plpgsql';
+        
+    SQL_O_MATIC
+    
+    execute "DROP FUNCTION IF EXISTS terrapop_areal_rasterization(bigint,bigint,bigint,bigint,integer)"
+    
+    execute cell_count_sql
+    execute rasterization_sql
+    
+  end
+end

data/db/migrate/20140908194021_create_area_data_rasters.rb

@@ -0,0 +1,39 @@
+class CreateAreaDataRasters < ActiveRecord::Migration
+  def change
+    
+    create_table :area_data_rasters do |t|
+      t.column     :sample_geog_level_id,  :bigint,  null: false
+      t.column     :raster_variable_id,    :bigint,  null: false
+      t.column     :area_data_variable_id, :bigint,  null: false
+      t.column     :raster_size,           :bigint,  null: false, default: 1
+      t.column     :valid,                 :boolean, null: false, default: true
+      t.column     :rast,                  :raster
+      t.timestamps
+    end
+    
+    add_index :area_data_rasters, :sample_geog_level_id
+    add_index :area_data_rasters, :raster_variable_id
+    add_index :area_data_rasters, :area_data_variable_id
+    
+    add_index :area_data_rasters, [:sample_geog_level_id, :raster_variable_id, :area_data_variable_id, :raster_size], unique: true, name: :area_data_rasters_uniq_index
+    
+    foreign_key :area_data_rasters, :sample_geog_level_id
+    foreign_key :area_data_rasters, :area_data_variable_id
+    foreign_key :area_data_rasters, :raster_variable_id
+    
+    #tiff_o_sql = <<-TIFF_O_MATIC
+    #CREATE OR REPLACE Function terrapop_get_area_level_raster_as_tiff(area_data_raster_id bigint)
+    #  Returns TABLE (geotiff bytea) as
+    #    $BODY$
+    #      BEGIN
+    #        RETURN
+    #          SELECT ST_AsTiff(adr.rast,'LZW',4326) AS rast_bytea FROM area_data_rasters AS adr WHERE id = area_data_raster_id;
+    #      END;
+    #    $BODY$
+    #    LANGUAGE plpgsql;
+    #TIFF_O_MATIC
+    
+    #execute tiff_o_sql
+    
+  end
+end

data/db/migrate/20140909143933_rename_valid_column_on_area_data_raster.rb

@@ -0,0 +1,5 @@
+class RenameValidColumnOnAreaDataRaster < ActiveRecord::Migration
+  def change
+    rename_column :area_data_rasters, :valid, :is_valid
+  end
+end

data/db/migrate/20140911201639_add_columns_to_area_data_raster.rb

@@ -0,0 +1,11 @@
+class AddColumnsToAreaDataRaster < ActiveRecord::Migration
+  def change
+    
+    add_column :area_data_rasters, :label,         :string, limit: 128
+    add_column :area_data_rasters, :mnemonic,      :string, limit: 32
+    add_column :area_data_rasters, :value,         :decimal, :precision => 64, :scale => 10
+    add_column :area_data_rasters, :cell_count,    :bigint
+    add_column :area_data_rasters, :pop_per_pixel, :decimal, :precision => 64, :scale => 10
+    
+  end
+end

data/db/migrate/20140911211923_alter_area_data_raster_indexes.rb

@@ -0,0 +1,12 @@
+class AlterAreaDataRasterIndexes < ActiveRecord::Migration
+  def change
+    remove_index  :area_data_rasters, name: :area_data_rasters_uniq_index
+    
+    add_column :area_data_rasters, :geog_instance_id, :bigint
+    
+    add_index :area_data_rasters, [:sample_geog_level_id, :raster_variable_id, :area_data_variable_id, :geog_instance_id, :raster_size], unique: true, name: :area_data_rasters_uniq_index
+    
+    foreign_key :area_data_rasters, :geog_instance_id
+    
+  end
+end

data/db/migrate/20140919144305_create_raster_variable_classifications.rb

@@ -0,0 +1,29 @@
+class CreateRasterVariableClassifications < ActiveRecord::Migration
+  def change
+    create_table :raster_variable_classifications do |t|
+      t.column    :raster_variable_id, :bigint
+      t.timestamps
+    end
+
+    add_column :raster_variables, :raster_variable_classification_id, :bigint
+    foreign_key :raster_variables, :raster_variable_classification_id
+    
+    add_index :raster_variables, :raster_variable_classification_id
+    add_index :raster_variable_classifications, :raster_variable_id
+    
+    foreign_key :raster_variable_classifications, :raster_variable_id
+    
+    # add_index :rasters, :raster_variable_id
+    
+  end
+end
+
+# A RasterVariable that is mosaic'd has many RasterVariables
+#
+#
+#
+#
+#
+#
+#
+#

data/db/migrate/20140925140810_add_other_raster_variables_to_raster_variable_classification.rb

@@ -0,0 +1,9 @@
+class AddOtherRasterVariablesToRasterVariableClassification < ActiveRecord::Migration
+  def change
+    add_column :raster_variable_classifications, :mosaic_raster_variable_id, :bigint
+    
+    add_index :raster_variable_classifications, :mosaic_raster_variable_id, name: :rvc_mosaic_raster_variable_id
+    foreign_key_raw :raster_variable_classifications, :mosaic_raster_variable_id, :raster_variables, :id
+    
+  end
+end

data/db/migrate/20140925172245_add_grouping_indicator_to_raster_classifications.rb

@@ -0,0 +1,5 @@
+class AddGroupingIndicatorToRasterClassifications < ActiveRecord::Migration
+  def change
+    add_column :raster_variable_classifications, :grouping, :bigint, default: nil
+  end
+end

data/db/migrate/20140925175157_add_classification_indicator_to_raster_variable.rb

@@ -0,0 +1,5 @@
+class AddClassificationIndicatorToRasterVariable < ActiveRecord::Migration
+  def change
+    add_column :raster_variables, :classification, :bigint, default: nil
+  end
+end

data/db/migrate/20140930142006_remove_boundary_id_from_geog_instance.rb

@@ -0,0 +1,6 @@
+class RemoveBoundaryIdFromGeogInstance < ActiveRecord::Migration
+  def change
+    remove_constraint_if_exists :geog_instances, :geog_instances_boundary_id_fkey
+    remove_column :geog_instances, :boundary_id
+  end
+end

data/db/migrate/20141003164150_add_terrapop_sample_id_to_map_and_geog_instance.rb

@@ -0,0 +1,11 @@
+class AddTerrapopSampleIdToMapAndGeogInstance < ActiveRecord::Migration
+  def change
+    
+    add_column :maps, :terrapop_sample_id, :bigint
+    add_column :geog_instances, :terrapop_sample_id, :bigint
+    
+    foreign_key :geog_instances, :terrapop_sample_id
+    foreign_key :maps, :terrapop_sample_id
+    
+  end
+end

data/db/migrate/20141029202813_add_nhgis_dat_file_to_sample_geog_level.rb

@@ -0,0 +1,5 @@
+class AddNhgisDatFileToSampleGeogLevel < ActiveRecord::Migration
+  def change
+    add_column :sample_geog_levels, :nhgis_dat_file, :string, limit: 254, null: true
+  end
+end

data/db/migrate/20141029204521_add_columns_to_sample_level_area_data_variable.rb

@@ -0,0 +1,6 @@
+class AddColumnsToSampleLevelAreaDataVariable < ActiveRecord::Migration
+  def change
+    add_column :sample_level_area_data_variables, :start_pos, :bigint, null: true
+    add_column :sample_level_area_data_variables, :width, :bigint, null: true
+  end
+end

data/db/migrate/20141104212652_create_categorical_raster_summarization.rb

@@ -0,0 +1,39 @@
+class CreateCategoricalRasterSummarization < ActiveRecord::Migration
+  def change
+
+    geog_instances = GeogInstance.where({}).limit(10)
+    raster_variables = RasterVariable.where({}).limit(1)
+
+    results_sql = <<-RASTER_O_MATIC
+SELECT clip.description, (clip.rd).value::integer as int_cat, upper(clip.mnemonic) || '_' || ((clip.rd).value::text) as cat, SUM((clip.rd).count) as numcells
+FROM (
+  select ST_ValueCount(ST_Clip(r.rast, p.geog::geometry)) as rd, p.description, p.code, rv.mnemonic
+  from boundaries p
+    INNER JOIN rasters r on ST_Intersects(r.rast, p.geog::geometry)
+    INNER JOIN raster_variables rv ON rv.id = r.raster_variable_id
+  WHERE p.geog_instance_id IN (#{geog_instances.map{|gi| gi.id}.join(", ")}) AND r.raster_variable_id IN (#{raster_variables.map{|rv| rv.id}.join(", ")})
+) clip
+WHERE (clip.rd).value IN (select DISTINCT (clip.rd).value::integer as all_LC
+      from(
+        SELECT ST_ValueCount(r.rast) as rd FROM boundaries p INNER JOIN rasters r on ST_Intersects(r.rast, p.geog::geometry) WHERE p.geog_instance_id IN (#{geog_instances.map{|gi| gi.id}.join(", ")}) AND r.raster_variable_id IN (#{raster_variables.map{|rv| rv.id}.join(", ")})
+  ) as lc
+ORDER BY all_LC)
+GROUP BY clip.description, (clip.rd).value, cat
+ORDER BY 1,2
+END;
+$$;
+RASTER_O_MATIC
+
+    category_sql = <<-RASTER_O_MATIC
+select DISTINCT upper(clip.mnemonic) || '_' || ((clip.rd).value::text) as cat, (clip.rd).value::integer as int_cat
+  from(
+    SELECT ST_ValueCount(r.rast) as rd, rv.mnemonic FROM boundaries p
+      INNER JOIN rasters r on ST_Intersects(r.rast, p.geog::geometry)
+      INNER JOIN raster_variables rv ON rv.id = r.raster_variable_id WHERE p.geog_instance_id IN (#{geog_instances.map{|gi| gi.id}.join(", ")}) AND r.raster_variable_id IN (#{raster_variables.map{|rv| rv.id}.join(", ")})
+  ) clip ORDER BY (clip.rd).value::integer
+END;
+$$;
+RASTER_O_MATIC
+  
+  end
+end

data/db/migrate/20141105202744_add_margin_file_to_sample_geog_level.rb

@@ -0,0 +1,5 @@
+class AddMarginFileToSampleGeogLevel < ActiveRecord::Migration
+  def change
+    add_column :sample_geog_levels, :nhgis_dat_file_m, :string, limit: 254, null: true
+  end
+end

data/db/migrate/20141105205347_add_margin_field_to_area_data_value.rb

@@ -0,0 +1,5 @@
+class AddMarginFieldToAreaDataValue < ActiveRecord::Migration
+  def change
+    add_column :area_data_values, :margin_value, :decimal, precision: 64, scale: 10, null: true
+  end
+end

data/db/migrate/20141105205607_add_join_fields_to_sample_geog_level.rb

@@ -0,0 +1,8 @@
+class AddJoinFieldsToSampleGeogLevel < ActiveRecord::Migration
+  def change
+    add_column :sample_geog_levels, :gisjoin1_start, :bigint, null: true
+    add_column :sample_geog_levels, :gisjoin1_width, :bigint, null: true
+    add_column :sample_geog_levels, :gisjoin2_start, :bigint, null: true
+    add_column :sample_geog_levels, :gisjoin2_width, :bigint, null: true
+  end
+end

data/db/migrate/20141202180411_add_new_categorical_summarization_function.rb

@@ -0,0 +1,62 @@
+class AddNewCategoricalSummarizationFunction < ActiveRecord::Migration
+  def change
+
+    sql = <<-SQL
+
+    CREATE OR REPLACE FUNCTION terrapop_categorical_raster_v0(IN sample_geog_lvl_id bigint, IN rasters_id bigint)
+      RETURNS TABLE(geog_instance_id bigint, geog_instance_label character varying, num_class bigint, mod_class double precision, total_area double precision) AS
+    $BODY$
+        DECLARE
+
+        BEGIN
+
+        RETURN QUERY
+        WITH poly_table AS (
+          SELECT sgl.id as sample_geog_level_id, gi.id as geog_instance_id, gi.label as geog_instance_label,
+          gi.code as geog_instance_code, ST_Transform(bound.geog::geometry, (SELECT ST_SRID(r.rast) FROM rasters r WHERE r.raster_variable_id = rasters_id LIMIT 1)) as geom
+          FROM sample_geog_levels sgl
+          inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+          inner join boundaries bound on bound.geog_instance_id = gi.id
+          WHERE sgl.id = sample_geog_lvl_id
+        ),
+         cat_rast AS
+        (
+        select p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code, ST_Union(ST_Clip(r.rast, p.geom)) as rast
+          from poly_table p inner join rasters r on ST_Intersects(r.rast, p.geom)
+          where r.raster_variable_id = rasters_id
+          group by p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code
+        ),
+        area_rast AS
+        (
+        select p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code, (ST_SummaryStats(ST_Union(ST_Clip(r.rast, p.geom)))).sum as categorical_area
+          from poly_table p inner join rasters r on ST_Intersects(r.rast, p.geom)
+          where r.raster_variable_id = ( select rv.area_reference_id from raster_variables rv where rv.id = rasters_id limit 1 )
+          group by p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code
+        )
+        SELECT area_rast.geog_instance_id, area_rast.geog_instance_label, cat_num_classes.number_of_classes, cat_mode_class.value as modal_class, area_rast.categorical_area 
+        FROM area_rast inner join 
+            (
+            SELECT cat_count.geog_instance_id, cat_count.geog_instance_label, count(cat_count.value) as number_of_classes
+            FROM (
+                select cat_rast.geog_instance_id, cat_rast.geog_instance_label, (ST_valuecount(cat_rast.rast)).value
+                from cat_rast
+            ) cat_count
+            GROUP BY cat_count.geog_instance_id, cat_count.geog_instance_label
+            ) cat_num_classes on (area_rast.geog_instance_id = cat_num_classes.geog_instance_id) inner join 
+            (
+            SELECT DISTINCT ON (cat_rast.geog_instance_id, cat_rast.geog_instance_label)
+            cat_rast.geog_instance_id, cat_rast.geog_instance_label, (ST_valuecount(cat_rast.rast)).*
+            FROM cat_rast
+            ORDER BY cat_rast.geog_instance_id, cat_rast.geog_instance_label, (ST_valuecount(cat_rast.rast)).count DESC
+            ) cat_mode_class ON (area_rast.geog_instance_id = cat_mode_class.geog_instance_id);
+
+        END;
+
+        $BODY$
+      LANGUAGE plpgsql VOLATILE;
+    SQL
+
+    execute sql
+
+  end
+end