geo_works-derivatives 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c40b9537076d66f4122e6436a4275d1e3bd26b43b4eebe86b8937709069e7c49
+  data.tar.gz: 713e155b39436bf6e2fe324ca5b033d4a8404e7cc27fc6e1e3ff204646e7e80d
+SHA512:
+  metadata.gz: 35b8607ddc30dc5f10fd8e0203a73c5746f362b1e0fa6f8d6c80ca0c9ee9654cfd16d39589fe2922ed42256abcb3be407bafa6e6785bdd0f8aeed9f54b157f4c
+  data.tar.gz: 3a9fb91fb9160dddef57b8a87606b6e6308361401b111d2ca6bd699db28022f685a12a4f50d949e8834f4068f1596f9c545fbb1e10f659dc3a0c1c4c7e1f95fb

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+.byebug_history
+# rspec failure tracking
+.rspec_status
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,22 @@
+inherit_gem:
+  bixby: bixby_default.yml
+AllCops:
+  DisplayCopNames: true
+  TargetRubyVersion: 2.3
+  Exclude:
+    - 'bin/*'
+    - 'vendor/**/*'
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+    - 'lib/geo_works/derivatives/processors/gdal.rb'
+    - 'lib/geo_works/derivatives/processors/image.rb'
+    - 'lib/geo_works/derivatives/specs/**/*'
+    - 'lib/tasks/**/*'
+    - 'db/schema.rb'
+    - 'Rakefile'
+    - '*.gemspec'
+Naming/FileName:
+  Exclude: 
+   - "geo_works-derivatives.gemspec"
+   - "Gemfile"

data/.travis.yml

@@ -0,0 +1,21 @@
+sudo: true
+language: ruby
+rvm:
+  - 2.4.5
+addons:
+  apt:
+    packages:
+    - libgdal1i
+    - libgdal-dev
+    - gdal-bin
+    - libcairo2-dev
+    - libpango1.0-dev
+before_install:
+ - git clone https://github.com/propublica/simple-tiles.git
+ - cd simple-tiles
+ - ./configure && make
+ - sudo make install
+ - cd ..
+ - gem install bundler -v 1.16.1
+before_script:
+  - gdalinfo --version

data/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in geo_derivatives.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,15 @@
+##########################################################################
+# Copyright 2018 Princeton University Library
+# Additional copyright may be held by others, as reflected in the commit log
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

data/README.md

@@ -0,0 +1,63 @@
+# GeoWorks::Derivatives
+
+[![Build Status](https://img.shields.io/travis/pulibrary/geo_works-derivatives/master.svg)](https://travis-ci.org/pulibrary/geo_works-derivatives)
+
+Hydra::Derivatives compatible processors for Geospatial derivative generation.
+
+## Dependencies
+
+1. [ImageMagick](http://www.imagemagick.org/)
+1. [GDAL](http://www.gdal.org/)
+    * You can install it on Mac OSX with `brew install gdal`.
+    * On Ubuntu, use `sudo apt-get install gdal-bin`.
+
+## Simple Tiles
+
+GeoWorks requires the image generation library [Simple Tiles](http://propublica.github.io/simple-tiles/).
+
+Mac OS X:
+
+- Install via Homebrew: ```brew install simple-tiles```
+
+Linux:
+
+- Install dependencies:
+
+  ```
+  libgdal-dev
+  libcairo2-dev
+  libpango1.0-dev
+  ```
+
+- Compile:
+
+  ```
+  $ git clone git@github.com:propublica/simple-tiles.git
+  $ cd simple-tiles
+  $ ./configure
+  $ make && make install
+  ```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'geo_works-derivatives'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install geo_works-derivatives
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/pulibrary/geo_works-derivatives.

data/Rakefile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'geo_works/derivatives'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/config/simpler_tiles.yml

@@ -0,0 +1,24 @@
+# Background color
+# Use '#ffffff' for a solid white background.
+# Add two zeros to a six digit hex color code
+# for a png (does not work with jpeg) image
+# with a transparent background.
+bg_color: '#ffffff00' 
+
+# The fill color for each shape
+fill: '#e4e3ea' 
+
+# The stroke color for each line
+stroke: '#483d8b'
+
+# Width of line in pixels
+weight: '0.3'
+
+# How lines are ended, one of: "butt", "round", "square"
+line_cap: 'square'
+
+# How lines are joined together, one of: "miter", "round", "bevel"
+line_join: 'miter'
+
+# For point rendering only, the radius in pixels of the circle
+radius: '2'

data/geo_works-derivatives.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "geo_works/derivatives/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = 'geo_works-derivatives'
+  spec.version       = GeoWorks::Derivatives::VERSION
+  spec.authors       = ['Eliot Jordan']
+  spec.email         = ['eliotj@princeton.edu']
+
+  spec.summary       = 'Geospatial derivative generation'
+  spec.description   = 'Hydra::Derivatives compatible processors for Geospatial derivative generation'
+  spec.homepage      = 'https://github.com/pulibrary/geo_works-derivatives'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise 'RubyGems 2.0 or newer is required to protect against ' \
+  #     'public gem pushes.'
+  # end
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'hydra-derivatives'
+  spec.add_dependency 'mime-types'
+  spec.add_dependency 'simpler-tiles'
+
+  spec.add_development_dependency 'bixby'
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'coveralls'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+end

data/lib/geo_works/derivatives/config.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+module GeoWorks
+  module Derivatives
+    module Config
+      class << self
+        def rendering_config
+          @config ||= RenderingConfiguration.new
+        end
+      end
+
+      class RenderingConfiguration
+        attr_reader :settings
+        attr_writer :bg_color,
+                    :fill,
+                    :stroke,
+                    :weight,
+                    :line_cap,
+                    :line_join,
+                    :radius
+
+        def initialize
+          @settings ||= File.exist?(config_file) ? YAML.load_file(config_file) : {}
+        end
+
+        def config_file
+          File.join(config_root_path, 'config/simpler_tiles.yml')
+        end
+
+        def config_root_path
+          if defined?(Rails) && Rails.root
+            Rails.root
+          else
+            Pathname.new(Dir.pwd)
+          end
+        end
+
+        def to_h
+          {
+            'stroke' => stroke,
+            'line-cap' => line_cap,
+            'line-join' => line_join,
+            'weight' => weight,
+            'fill' => fill,
+            'radius' => radius
+          }
+        end
+
+        def bg_color
+          @bg_color ||= settings.fetch('bg_color', '#ffffff00')
+        end
+
+        def fill
+          @fill ||= settings.fetch('fill', '#e4e3ea')
+        end
+
+        def stroke
+          @stroke ||= settings.fetch('stroke', '#483d8b')
+        end
+
+        def weight
+          @weight ||= settings.fetch('weight', '0.3')
+        end
+
+        def line_cap
+          @line_cap ||= settings.fetch('line_cap', 'square')
+        end
+
+        def line_join
+          @line_join ||= settings.fetch('line_join', 'miter')
+        end
+
+        def radius
+          @radius ||= settings.fetch('radius', '2')
+        end
+      end
+    end
+  end
+end

data/lib/geo_works/derivatives/processors/base_geo_processor.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+module GeoWorks
+  module Derivatives
+    module Processors
+      module BaseGeoProcessor
+        extend ActiveSupport::Concern
+
+        included do
+          # Chains together and recursively runs a set of commands.
+          # Except for the last command in the queue, a temp file
+          # is created as output and is then fed into the input of the next.
+          # Temp files are deleted in reverse order after the last command
+          # is run. The commands must have the same method signature:
+          # command_name(in_path, out_path, options)
+          #
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param method_queue [Array] set of commands to run
+          # @param options [Hash] creation options to pass
+          # rubocop:disable Metrics/MethodLength
+          def self.run_commands(in_path, out_path, method_queue, options)
+            next_step = method_queue.shift
+            temp = temp_path(out_path)
+            begin
+              if method_queue.empty?
+                method(next_step).call(in_path, out_path, options)
+              else
+                method(next_step).call(in_path, temp, options)
+                run_commands(temp, out_path, method_queue, options)
+                FileUtils.rm_rf(temp)
+              end
+            rescue => e
+              FileUtils.rm_rf(in_path) if Dir.exist?(in_path)
+              FileUtils.rm_rf(temp) if Dir.exist?(temp)
+              raise e
+            end
+          end
+          # rubocop:enable Metrics/MethodLength
+
+          # Returns a path to an intermediate temp file or directory.
+          # @param path [String] input file path to base temp path on
+          # @return [String] tempfile path
+          def self.temp_path(path)
+            time = (Time.now.to_f * 1000).to_i
+            "#{File.dirname(path)}/#{File.basename(path, File.extname(path))}_#{time}"
+          end
+        end
+
+        def options_for(_format)
+          {
+            label: label,
+            output_size: output_size,
+            output_srid: output_srid,
+            basename: basename,
+            id: id
+          }
+        end
+
+        # Returns the label directive or an empty string.
+        # @return [Sting] output label
+        def label
+          directives.fetch(:label, '')
+        end
+
+        # Transforms the size directive into a GDAL size parameter.
+        # @return [String] derivative size
+        def output_size
+          return unless directives[:size]
+          directives[:size].tr('x', ' ')
+        end
+
+        # Gets srid for reprojection derivative or returns WGS 84.
+        # @return [String] spatial reference code
+        def output_srid
+          directives.fetch(:srid, 'EPSG:4326')
+        end
+
+        # Extracts the base file name (without extension) from the source file path.
+        # @return [String] base file name for source
+        def basename
+          File.basename(source_path, File.extname(source_path))
+        end
+
+        # Gets the fileset id or returns nil.
+        # @return [String] fileset id
+        def id
+          directives.fetch(:id, nil)
+        end
+      end
+    end
+  end
+end

data/lib/geo_works/derivatives/processors/gdal.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module GeoWorks
+  module Derivatives
+    module Processors
+      module Gdal
+        extend ActiveSupport::Concern
+
+        included do
+          # Executes a gdal_translate command. Used to translate a raster
+          # format into a different format. Also used in generating thumbnails
+          # from vector data.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param options [Hash] creation options
+          def self.translate(in_path, out_path, _options)
+            execute "gdal_translate -q -ot Byte -of GTiff -co TILED=YES -expand rgb -co COMPRESS=DEFLATE \"#{in_path}\" #{out_path}"
+          rescue StandardError
+            # Try without expanding rgb
+            execute "gdal_translate -q -ot Byte -of GTiff -co TILED=YES -co COMPRESS=DEFLATE \"#{in_path}\" #{out_path}"
+          end
+
+          # Executes a gdalwarp command. Used to transform a raster
+          # from one projection into another.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param options [Hash] creation options
+          def self.warp(in_path, out_path, options)
+            execute "gdalwarp -q -t_srs #{options[:output_srid]} "\
+                    "#{in_path} #{out_path} -co TILED=YES -co COMPRESS=NONE"
+          end
+
+          # Executes a gdal_translate command. Used to compress
+          # a previously uncompressed raster.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param options [Hash] creation options
+          def self.compress(in_path, out_path, options)
+            execute "gdal_translate -q -ot Byte -a_srs #{options[:output_srid]} "\
+                      "#{in_path} #{out_path} -co COMPRESS=JPEG -co JPEG_QUALITY=90"
+          end
+
+          # Executes gdaladdo and gdal_translate commands. Used to add internal overviews
+          # and then compress a previously uncompressed raster.
+          # Output will be a Cloud Optimized GeoTIFF.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param options [Hash] creation options
+          def self.cloud_optimized_geotiff(in_path, out_path, _options)
+            system("gdaladdo -q -r average #{in_path} 2 4 8 16 32")
+            execute("gdal_translate -q -expand rgb #{in_path} #{out_path} -co TILED=YES "\
+                      "-co COMPRESS=JPEG -co COPY_SRC_OVERVIEWS=YES")
+          rescue StandardError
+            # Try without expanding rgb
+            execute("gdal_translate -q #{in_path} #{out_path} -co TILED=YES "\
+                      "-co COMPRESS=JPEG -co COPY_SRC_OVERVIEWS=YES")
+          end
+
+          # Executes a gdal_rasterize command. Used to rasterize vector
+          # format into raster format.
+          # @param in_path [String] file input path
+          # #param options [Hash] creation options
+          # @param out_path [String] processor output file path
+          def self.rasterize(in_path, out_path, options)
+            execute "gdal_rasterize -q -burn 0 -init 255 -ot Byte -ts "\
+                      "#{options[:output_size]} -of GTiff #{in_path} #{out_path}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/geo_works/derivatives/processors/image.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+require 'mini_magick'
+
+module GeoWorks
+  module Derivatives
+    module Processors
+      module Image
+        extend ActiveSupport::Concern
+        included do
+          # Uses imagemagick to resize an image and convert it to the output format.
+          # Keeps the aspect ratio of the original image and adds padding to
+          # to the output image. The file extension is the output format.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path.
+          # @param options [Hash] creation options
+          # @option options [String] `:output_size` as "w h" or "wxh"
+          def self.convert(in_path, out_path, options)
+            size = options[:output_size].tr(' ', 'x')
+            convert = MiniMagick::Tool::Convert.new(whiny: false)
+            convert << in_path
+            convert << "-resize"
+            convert << size
+            convert << "-extent"
+            convert << size
+            convert << "-background"
+            convert << "white"
+            convert << "-gravity"
+            convert << "center"
+            convert << out_path
+
+            # suppress stderr b/c geotiffs return 'unknown field' warnings
+            convert.call { |_stdout, _stderr| }
+          end
+
+          # Trims extra whitespace.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path.
+          # @param options [Hash] creation options
+          def self.trim(in_path, out_path, _options)
+            MiniMagick::Tool::Convert.new do |convert|
+              convert << in_path
+              convert << "-trim"
+              convert << out_path
+            end
+          end
+
+          # Centers and pads image.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path.
+          # @param options [Hash] creation options
+          def self.center(in_path, out_path, options)
+            MiniMagick::Tool::Convert.new do |convert|
+              convert << "-size"
+              convert << options[:output_size].tr(' ', 'x')
+              convert << "xc:white"
+              convert << in_path
+              convert << "-gravity"
+              convert << "center"
+              convert << "-composite"
+              convert << out_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/geo_works/derivatives/processors/ogr.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+module GeoWorks
+  module Derivatives
+    module Processors
+      module Ogr
+        extend ActiveSupport::Concern
+
+        included do
+          # Executes a ogr2ogr command. Used to reproject a
+          # vector dataset and save the output as a shapefile
+          # @param in_path [String] file input path
+          # #param options [Hash] creation options
+          # @param out_path [String] processor output file path
+          def self.reproject(in_path, out_path, options)
+            execute "env SHAPE_ENCODING= ogr2ogr -q -nln #{options[:id]} -f 'ESRI Shapefile'"\
+                      " -t_srs #{options[:output_srid]} -preserve_fid '#{out_path}' '#{in_path}'"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/geo_works/derivatives/processors/raster/aig.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+module GeoWorks
+  module Derivatives
+    module Processors
+      module Raster
+        class Aig < GeoWorks::Derivatives::Processors::Raster::Base
+          include GeoWorks::Derivatives::Processors::Zip
+
+          def self.encode(path, options, output_file)
+            unzip(path, output_file) do |zip_path|
+              info = Info.new(zip_path)
+              options[:min_max] = info.min_max
+              case options[:label]
+              when :thumbnail
+                encode_raster(zip_path, output_file, options)
+              when :display_raster
+                reproject_raster(zip_path, output_file, options)
+              end
+            end
+          end
+
+          # Set of commands to run to reproject the AIG.
+          # @return [Array] set of command name symbols
+          def self.reproject_queue
+            [:warp, :translate, :compress]
+          end
+
+          # Executes a gdal_translate command to translate a raster
+          # format into a different format with a scaling options. This command
+          # scales the min and max values of the raster into the 0 to 255 range.
+          # Scale is inverted (255 to 0) to create a better visualization.
+          # @param in_path [String] file input path
+          # @param out_path [String] processor output file path
+          # @param options [Hash] creation options
+          def self.translate(in_path, out_path, options)
+            execute "gdal_translate -scale #{options[:min_max]} 255 0 "\
+                      "-q -ot Byte -of GTiff \"#{in_path}\" #{out_path}"
+          end
+        end
+      end
+    end
+  end
+end