terrapop_models 0.1.0

data/db/migrate/20150528141915_yancat_to_bin_summarization_function.rb

@@ -0,0 +1,66 @@
+class YancatToBinSummarizationFunction < ActiveRecord::Migration
+  def change
+    sql =<<-SQL
+    CREATE OR REPLACE FUNCTION terrapop_reclassify_categorical_raster_to_binary_summariz_v3(IN sample_geog_lvl_id bigint, IN rasters_id bigint, IN bnd_num integer)
+        RETURNS TABLE(geog_instance bigint, geog_instance_label character varying, pixel_count bigint, binary_area double precision, percent_area double precision, total_area double precision) AS
+        $BODY$
+
+    DECLARE
+        area_raster_id integer;
+        categorical_raster_id integer;
+
+    BEGIN
+        SELECT rv.area_reference_id as area_id into area_raster_id FROM raster_variables rv WHERE rv.id =  rasters_id ;
+        SELECT rv.second_area_reference_id into categorical_raster_id FROM raster_variables rv WHERE rv.id = rasters_id ;
+
+        RETURN QUERY
+
+        WITH lookup AS
+            (
+            SELECT replace(replace(array_agg(classification::text || ':1')::text, '{', ''), '}', '') as exp
+            FROM raster_variables WHERE id IN (
+                    select raster_variable_classifications.mosaic_raster_variable_id 
+                    from raster_variable_classifications
+                    where raster_variable_classifications.raster_variable_id = rasters_id)
+            ), transformation as
+            (
+            SELECT ST_SRID(r.rast) as prj_value
+            FROM rasters r
+            WHERE r.raster_variable_id = categorical_raster_id
+            LIMIT 1
+            ), polygon as
+            (
+            SELECT sgl.id as sample_geog_level_id, gi.id as geog_instance_id, gi.label as geog_instance_label,
+            gi.code as geog_instance_code, ST_Transform(bound.geog::geometry, t.prj_value) as geom
+            FROM transformation t, sample_geog_levels sgl
+            inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+            inner join boundaries bound on bound.geog_instance_id = gi.id
+            WHERE sgl.id = sample_geog_lvl_id
+            ), r_table as
+            (
+            SELECT p.geog_instance_id, p.geog_instance_label, ST_union(ST_Reclass(ST_Clip(r.rast, bnd_num ,p.geom, TRUE),1,l.exp, '8BUI',0)) as rast
+            FROM lookup l, polygon p inner join rasters r on ST_Intersects(r.rast, p.geom)
+            WHERE r.raster_variable_id = categorical_raster_id
+            GROUP by p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code
+            ), a_table as
+            (
+            SELECT p.geog_instance_id, p.geog_instance_label, ST_union(ST_Clip(r.rast, p.geom)) as rast
+            FROM lookup l, polygon p inner join rasters r on ST_Intersects(r.rast, p.geom)
+            WHERE r.raster_variable_id = area_raster_id
+            GROUP by p.sample_geog_level_id, p.geog_instance_id, p.geog_instance_label, p.geog_instance_code
+            ), calc as
+            (
+            SELECT r.geog_instance_id, r.geog_instance_label, (ST_SummaryStats(r.rast)).*, (ST_SummaryStats(a.rast)).sum as total_area,
+            (ST_SummaryStats(ST_MapAlgebra(r.rast, 1, a.rast, 1, '[rast1]*[rast2]', '32BUI'))).sum as binary_area
+            FROM r_table r inner join a_table a on (r.geog_instance_id = a.geog_instance_id)
+            )
+            SELECT c.geog_instance_id, c.geog_instance_label, c.count as pixel_count, c.binary_area,
+            c.binary_area / c.total_area as percent_area, c.total_area as total_area
+            FROM calc c;
+
+    END;
+    $BODY$
+    LANGUAGE plpgsql VOLATILE;
+    SQL
+  end
+end

data/db/migrate/20150529182542_add_indexes_to_geog_instances.rb

@@ -0,0 +1,9 @@
+class AddIndexesToGeogInstances < ActiveRecord::Migration
+  def change
+    add_index :geog_instances, :parent_id
+    add_index :geog_instances, :terrapop_sample_id
+    add_index :geog_instances, :code
+    add_index :geog_instances, :sample_geog_level_id
+    
+  end
+end

data/db/migrate/20150601160855_drop_area_data_variable_availability_caches.rb

@@ -0,0 +1,5 @@
+class DropAreaDataVariableAvailabilityCaches < ActiveRecord::Migration
+  def change
+    drop_table :area_data_variable_availability_caches
+  end
+end

data/db/migrate/20150625150152_add_geog_instance_f_key_on_boundaries.rb

@@ -0,0 +1,5 @@
+class AddGeogInstanceFKeyOnBoundaries < ActiveRecord::Migration
+  def change
+    foreign_key :boundaries, :geog_instance_id
+  end
+end

data/db/migrate/20150626144305_geog_instance_geog_code_wider.rb

@@ -0,0 +1,6 @@
+class GeogInstanceGeogCodeWider < ActiveRecord::Migration
+  def change
+    change_column :geog_instances, :geog_code, :string, limit: 64
+    change_column :geog_instances, :str_code,  :string, limit: 64
+  end
+end

data/db/migrate/20150709161201_glc_summarization_version5.rb

@@ -0,0 +1,79 @@
+class GlcSummarizationVersion5 < ActiveRecord::Migration
+  def change
+    sql =<<-SQL
+      CREATE OR REPLACE FUNCTION terrapop_glc_binary_summarization_v5(IN sample_geog_lvl_id bigint, IN rasters_id bigint)
+          RETURNS TABLE(geog_instance_id bigint, geog_instance_label character varying, binary_area double precision, total_area double precision, percent_area double precision) AS
+            $BODY$
+            BEGIN
+            RETURN QUERY
+        WITH srid as
+
+        (
+
+        	SELECT st_srid(r.rast) as srid FROM rasters r
+
+        	where r.raster_variable_id = rasters_id
+
+        	limit 1
+
+        ),
+
+       bounds as
+        (
+
+        	SELECT sgl.id as sample_geog_level_id, gi.id as gid, gi.label as label, gi.code as geog_instance_code, st_transform(bound.geog::geometry, srid.srid) as geom
+
+        	FROM srid, sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id inner join boundaries bound on bound.geog_instance_id = gi.id
+
+        	where sgl.id = sample_geog_lvl_id group by sgl.id, gi.id, gi.label, gi.code, bound.geog, srid.srid
+
+        ),
+		
+    		bin_rast as 
+    		(
+    			SELECT b.gid as gid, b.label as label, ST_Union(ST_Clip(r.rast, b.geom)) as rast
+    			FROM bounds b inner join rasters r on ST_Intersects(r.rast, b.geom)
+    			where r.raster_variable_id = rasters_id group by b.gid, b.label
+    		),
+
+    		area_reference_id as (select rv.area_reference_id as id from raster_variables rv where rv.id = rasters_id limit 1),
+
+    		area_ref_rast as
+    		(
+    			SELECT b.gid as gid, b.label as label, ST_Union(ST_Clip(r.rast, b.geom)) as rast
+    			FROM area_reference_id a, bounds b inner join rasters r on ST_Intersects(r.rast, b.geom)
+    			where r.raster_variable_id = a.id group by b.gid, b.label
+    		),
+
+    		binary_area as
+    		(
+    			select b.gid, b.label, (ST_SummaryStats(ST_MapAlgebra(b.rast, 1, a.rast, 1, '[rast1]*[rast2]'))).sum as binary_area
+    			FROM bin_rast b inner join area_ref_rast a on b.gid = a.gid
+    			group by b.gid, b.label, a.rast, b.rast
+    		),
+
+    		total_area as
+    		(
+    			SELECT a.gid as gid, a.label as label, (ST_SummaryStats(a.rast, true)).sum as total_area
+    			FROM area_ref_rast a group by a.gid, a.label, a.rast order by a.label
+    		),
+
+    		percent_area as
+    		(
+    			select a.gid as gid, a.label as label, a.binary_area/b.total_area as percent_area
+    			from binary_area a, total_area b where a.gid = b.gid order by a.label
+    		)
+
+    		SELECT b.gid, b.label, b.binary_area, t.total_area, p.percent_area
+    		FROM binary_area b inner join total_area t on b.gid = t.gid inner join percent_area p on t.gid = p.gid;
+
+            END;
+            $BODY$
+      LANGUAGE plpgsql VOLATILE
+      COST 100
+      ROWS 1000;
+    SQL
+    
+    execute(sql)
+  end
+end

data/db/migrate/20150713165442_glc_summarization_version6.rb

@@ -0,0 +1,82 @@
+class GlcSummarizationVersion6 < ActiveRecord::Migration
+  def change
+    sql =<<-SQL
+    CREATE OR REPLACE FUNCTION terrapop_glc_binary_summarization_v6(IN sample_geog_lvl_id bigint, IN rasters_id bigint)
+        RETURNS TABLE(geog_instance_id bigint, geog_instance_label character varying, binary_area double precision, total_area double precision, percent_area double precision) AS
+          $BODY$
+          BEGIN
+          RETURN QUERY
+      WITH srid as
+
+      (
+
+      	SELECT st_srid(r.rast) as srid FROM rasters r
+
+      	where r.raster_variable_id = rasters_id
+
+      	limit 1
+
+      ),
+      only_rasters AS 
+      (
+        SELECT id, raster_variable_id, rast FROM rasters WHERE raster_variable_id = rasters_id
+      ),
+      bounds as
+      (
+
+      	SELECT sgl.id as sample_geog_level_id, gi.id as gid, gi.label as label, gi.code as geog_instance_code, st_transform(bound.geog::geometry, srid.srid) as geom
+
+      	FROM srid, sample_geog_levels sgl inner join geog_instances gi on sgl.id = gi.sample_geog_level_id inner join boundaries bound on bound.geog_instance_id = gi.id
+
+      	where sgl.id = sample_geog_lvl_id group by sgl.id, gi.id, gi.label, gi.code, bound.geog, srid.srid
+
+      ),
+	
+  		bin_rast as 
+  		(
+  			SELECT b.gid as gid, b.label as label, ST_Union(ST_Clip(r.rast, b.geom)) as rast
+  			FROM bounds b inner join only_rasters r on ST_Intersects(r.rast, b.geom)
+  			where r.raster_variable_id = rasters_id group by b.gid, b.label
+  		),
+
+  		area_reference_id as (select rv.area_reference_id as id from raster_variables rv where rv.id = rasters_id limit 1),
+
+  		area_ref_rast as
+  		(
+  			SELECT b.gid as gid, b.label as label, ST_Union(ST_Clip(r.rast, b.geom)) as rast
+  			FROM area_reference_id a, bounds b inner join only_rasters r on ST_Intersects(r.rast, b.geom)
+  			where r.raster_variable_id = a.id group by b.gid, b.label
+  		),
+
+  		binary_area as
+  		(
+  			select b.gid, b.label, (ST_SummaryStats(ST_MapAlgebra(b.rast, 1, a.rast, 1, '[rast1]*[rast2]'))).sum as binary_area
+  			FROM bin_rast b inner join area_ref_rast a on b.gid = a.gid
+  			group by b.gid, b.label, a.rast, b.rast
+  		),
+
+  		total_area as
+  		(
+  			SELECT a.gid as gid, a.label as label, (ST_SummaryStats(a.rast, true)).sum as total_area
+  			FROM area_ref_rast a group by a.gid, a.label, a.rast order by a.label
+  		),
+
+  		percent_area as
+  		(
+  			select a.gid as gid, a.label as label, a.binary_area/b.total_area as percent_area
+  			from binary_area a, total_area b where a.gid = b.gid order by a.label
+  		)
+
+  		SELECT b.gid, b.label, b.binary_area, t.total_area, p.percent_area
+  		FROM binary_area b inner join total_area t on b.gid = t.gid inner join percent_area p on t.gid = p.gid;
+
+          END;
+          $BODY$
+    LANGUAGE plpgsql VOLATILE
+    COST 100
+    ROWS 1000;
+    SQL
+    
+    execute(sql)
+  end
+end

data/db/migrate/20150713185652_tp_jpeg_raster_clip_color_map_v2.rb

@@ -0,0 +1,34 @@
+class TpJpegRasterClipColorMapV2 < ActiveRecord::Migration
+  def change
+    sql = <<-SQL
+      CREATE OR REPLACE Function terrapop_jpeg_raster_clip_colormap_v2(sample_geog_lvl_id bigint, rasters_id bigint, raster_bnd integer, colormap text)
+      RETURNS TABLE (tiff bytea) AS
+      $BODY$
+
+        BEGIN
+        RETURN QUERY
+
+        WITH poly_table AS 
+        (
+        SELECT ST_Buffer(ST_Transform(bound.geog::geometry, (SELECT ST_SRID(r.rast) FROM rasters r WHERE r.raster_variable_id = rasters_id LIMIT 1)), 0) as geom
+          FROM sample_geog_levels sgl
+          inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+          inner join boundaries bound on bound.geog_instance_id = gi.id
+          where gi.sample_geog_level_id = sample_geog_lvl_id
+        ),
+        new_rast AS
+        (
+          SELECT ST_Union(ST_Clip(r.rast, raster_bnd, p.geom, TRUE)) as rast
+          FROM poly_table p inner join rasters r on ST_Intersects(r.rast,p.geom)
+          where r.raster_variable_id = rasters_id
+        )
+        select ST_AsJPEG(ST_ColorMap(ST_Union(ST_Rescale(r.rast, 0.5)), 1, colormap), 1) as tiff from new_rast r;
+
+        END;
+      $BODY$
+      LANGUAGE 'plpgsql';
+    SQL
+    
+    execute sql
+  end
+end

data/db/migrate/20150716182438_new_rasters_table.rb

@@ -0,0 +1,18 @@
+class NewRastersTable < ActiveRecord::Migration
+  def change
+    create_table :rasters_new do |t|
+      t.column    :raster_variable_id, :bigint
+      t.column    :table_name, :text
+      t.column    :area_reference_id,  :bigint
+      t.column    :second_area_reference_id, :bigint
+      t.column    :r_table_schema, :text
+      t.timestamps
+    end
+
+    add_index :rasters_new, :raster_variable_id
+    add_index :rasters_new, :area_reference_id
+    add_index :rasters_new, :second_area_reference_id
+
+    foreign_key :rasters_new, :raster_variable_id
+  end
+end

data/db/migrate/20150716184828_new_stored_procedures_for_new_rasters.rb

@@ -0,0 +1,70 @@
+class NewStoredProceduresForNewRasters < ActiveRecord::Migration
+  def change
+    sql=<<-SQL
+     CREATE OR REPLACE FUNCTION make_jpeg2(IN sample_geog_lvl_id integer, IN rast_id integer, IN rast_band_num integer)
+  RETURNS SETOF raster AS
+$BODY$
+DECLARE
+
+    one_raster text := 'hello';
+    query text := '';
+
+    BEGIN
+
+    WITH lookup AS
+    (
+    SELECT replace(replace(array_agg(classification::text || ':1')::text, '{', ''), '}', '') as exp
+    FROM raster_variables WHERE id IN (
+            select raster_variable_classifications.mosaic_raster_variable_id
+            from raster_variable_classifications
+            where raster_variable_classifications.raster_variable_id =  rast_id )
+    ), cat_rast as
+    (
+    SELECT rv.second_area_reference_id as cat_id
+    FROM raster_variables rv
+    WHERE rv.id =  rast_id
+    )
+    SELECT r_table_schema || '.' || table_name as tablename
+    INTO one_raster
+    FROM cat_rast inner join rasters_new on cat_rast.cat_id = rasters_new.raster_variable_id ;
+
+    RAISE NOTICE  ' % ', one_raster;
+
+    query := $$ WITH lookup AS
+    (
+    SELECT replace(replace(array_agg(classification::text || ':1')::text, '{', ''), '}', '') as exp
+    NTOFROM raster_variables WHERE id IN (
+            select raster_variable_classifications.mosaic_raster_variable_id
+            from raster_variable_classifications
+            where raster_variable_classifications.raster_variable_id = $$ || rast_id || $$)
+    )
+    , transformation as
+    (
+    SELECT ST_SRID(r.rast) as prj_value
+    FROM $$ || one_raster || $$ r
+    LIMIT 1
+    ), polygon as
+    (
+    SELECT sgl.id as sample_geog_level_id, gi.id as geog_instance_id, gi.label as geog_instance_label,
+    gi.code as geog_instance_code, ST_Transform(bound.geog::geometry, t.prj_value) as geom
+    FROM transformation t, sample_geog_levels sgl
+    inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+    inner join boundaries bound on bound.geog_instance_id = gi.id
+    WHERE sgl.id = $$ || sample_geog_lvl_id || $$
+    )
+    SELECT ST_union(ST_Reclass(ST_Clip(r.rast, 1,p.geom, TRUE),1,l.exp, '8BUI',0)) as rast
+    FROM lookup l, polygon p inner join $$ || one_raster || $$ r on ST_Intersects(r.rast,p.geom) $$ ;
+
+    RAISE NOTICE  ' % ', query;
+
+    RETURN QUERY execute query;
+
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100
+  ROWS 1000;
+    SQL
+    execute sql
+  end
+end

data/db/migrate/20150721161943_remove_ownership_statement_from_function.rb

@@ -0,0 +1,69 @@
+class RemoveOwnershipStatementFromFunction < ActiveRecord::Migration
+  def change
+    sql=<<-SQL
+     CREATE OR REPLACE FUNCTION make_jpeg2(IN sample_geog_lvl_id integer, IN rast_id integer, IN rast_band_num integer)
+  RETURNS SETOF raster AS
+$BODY$
+DECLARE
+    one_raster text := 'hello';
+    query text := '';
+
+    BEGIN
+
+    WITH lookup AS
+    (
+    SELECT replace(replace(array_agg(classification::text || ':1')::text, '{', ''), '}', '') as exp
+    FROM raster_variables WHERE id IN (
+            select raster_variable_classifications.mosaic_raster_variable_id
+            from raster_variable_classifications
+            where raster_variable_classifications.raster_variable_id =  rast_id )
+    ), cat_rast as
+    (
+    SELECT rv.second_area_reference_id as cat_id
+    FROM raster_variables rv
+    WHERE rv.id =  rast_id
+    )
+    SELECT r_table_schema || '.' || table_name as tablename
+    INTO one_raster
+    FROM cat_rast inner join rasters_new on cat_rast.cat_id = rasters_new.raster_variable_id ;
+
+    RAISE NOTICE  ' % ', one_raster;
+
+    query := $$ WITH lookup AS
+    (
+    SELECT replace(replace(array_agg(classification::text || ':1')::text, '{', ''), '}', '') as exp
+    NTOFROM raster_variables WHERE id IN (
+            select raster_variable_classifications.mosaic_raster_variable_id
+            from raster_variable_classifications
+            where raster_variable_classifications.raster_variable_id = $$ || rast_id || $$)
+    )
+    , transformation as
+    (
+    SELECT ST_SRID(r.rast) as prj_value
+    FROM $$ || one_raster || $$ r
+    LIMIT 1
+    ), polygon as
+    (
+    SELECT sgl.id as sample_geog_level_id, gi.id as geog_instance_id, gi.label as geog_instance_label,
+    gi.code as geog_instance_code, ST_Transform(bound.geog::geometry, t.prj_value) as geom
+    FROM transformation t, sample_geog_levels sgl
+    inner join geog_instances gi on sgl.id = gi.sample_geog_level_id
+    inner join boundaries bound on bound.geog_instance_id = gi.id
+    WHERE sgl.id = $$ || sample_geog_lvl_id || $$
+    )
+    SELECT ST_union(ST_Reclass(ST_Clip(r.rast, 1,p.geom, TRUE),1,l.exp, '8BUI',0)) as rast
+    FROM lookup l, polygon p inner join $$ || one_raster || $$ r on ST_Intersects(r.rast,p.geom) $$ ;
+
+    RAISE NOTICE  ' % ', query;
+
+    RETURN QUERY execute query;
+
+END;
+$BODY$
+  LANGUAGE plpgsql VOLATILE
+  COST 100
+  ROWS 1000;
+    SQL
+    execute sql
+  end
+end