zooniverse_data 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 58529b2e92c6d615e55e168226d6532a0490626c
+  data.tar.gz: c0698d676a7449cb6d5c3f59e17e71576ddfb50d
+SHA512:
+  metadata.gz: 43ae0c3c057c6f129ce6551dda2cc8e43ad16bb7c90e5d0ef07c4079f10d8711358e13204edb8c81506b0fe0fdef515388a488b4287b4580edf9cfac9a0d50df
+  data.tar.gz: 66644c38072d4c70376e671fd65b158e2ad92d859a71a706b88ceba4ece0eaff910d6ed3f7580b4671623d79a58a3ec17f2c2576892dd57c9037a1f457f91142

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.DS_Store

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Michael Parrish
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# ZooniverseData
+
+## This gem is useless unless you have access to [Ouroboros](https://github.com/zooniverse/ouroboros)
+
+The primary responsibilities of this gem are to
+
+  - Create JSON manifests for projects
+  - Customize subjects and groups being created in Ouroboros
+  - Customize data products used in projects
+
+## Installation
+
+Install [ImageMagick](http://www.imagemagick.org)
+
+Add this line to your application's Gemfile:
+
+    gem 'zooniverse_data'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install zooniverse_data
+
+## Usage
+
+This gem is required and used from within [Ouroboros](https://github.com/zooniverse/ouroboros)
+
+## Contributing
+
+1. Fork it ( https://github.com/zooniverse/zooniverse_data/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/lib/zooniverse_data/helpers/base.rb

@@ -0,0 +1,54 @@
+module ZooniverseData
+  module Helpers
+    module Base
+      extend ActiveSupport::Concern
+      
+      included do
+        attr_accessor :manifest
+        attr_accessor :entry
+      end
+      
+      def customize(manifest: manifest, entry: entry)
+        self.manifest = manifest
+        self.entry = entry
+        
+        if entry.subject?
+          customize_subject
+        elsif entry.group?
+          customize_group
+        end
+      end
+      
+      def customize_subject
+        
+      end
+      
+      def customize_group
+        
+      end
+      
+      def each_location
+        new_locations = { }
+        entry.location.each_pair do |key, value|
+          new_locations[key] = if value.is_a?(Array)
+            value.collect do |location|
+              _new_location_from yield(key, location)
+            end
+          else
+            _new_location_from yield(key, value)
+          end
+        end
+        
+        set_location new_locations
+      end
+      
+      def set_location(hash)
+        entry.update :$set => { location: hash }
+      end
+      
+      def _new_location_from(result)
+        result.respond_to?(:path) ? result.path : result
+      end
+    end
+  end
+end

data/lib/zooniverse_data/helpers/images.rb

@@ -0,0 +1,174 @@
+require 'fastimage'
+require 'ostruct'
+
+module ZooniverseData
+  module Helpers
+    module Images
+      class ImageConversionError < StandardError; end
+      class ImageOptimizationError < StandardError; end
+      
+      class Image
+        attr_accessor :path, :raise_exceptions
+        
+        def initialize(path: path, raise_exceptions: true)
+          self.path = path
+          self.raise_exceptions = raise_exceptions
+        end
+        
+        def size
+          width, height = info.size
+          OpenStruct.new width: width, height: height
+        end
+        
+        def type
+          info.type
+        end
+        
+        def info
+          @info ||= FastImage.new(path, raise_on_failure: raise_exceptions)
+        end
+        
+        def optimize
+          tap do
+            case type
+            when :jpeg
+              _run_optimization "jpegtran -copy none -optimize -progressive -outfile '#{ tempfile.path }' '#{ path }'"
+            when :png, :bmp, :gif, :tiff
+              _run_optimization "optipng -strip all -o2 -quiet '#{ path }'"
+            end
+          end
+        end
+        
+        def tempfile
+          return @tempfile if @tempfile
+          @tempfile = Tempfile.new File.basename path
+        end
+        
+        def remove_tempfile
+          tempfile.delete
+          @tempfile = nil
+        end
+        
+        def replace_with_tempfile
+          `cp '#{ tempfile.path }' '#{ path }'`
+          remove_tempfile
+        end
+        
+        def _run_optimization(command)
+          success = system command
+          raise ImageOptimizationError.new('Image optimization failed') unless success
+          replace_with_tempfile if @tempfile
+        end
+      end
+      
+      class Converter
+        attr_accessor :input_image, :output_image, :flags
+        attr_accessor :raise_exceptions, :remove_original, :optimize
+        
+        def initialize(path: path, raise_exceptions: true, remove_original: true, optimize: true)
+          self.input_image = Image.new path: path, raise_exceptions: raise_exceptions
+          self.remove_original = remove_original
+          self.optimize = optimize
+          self.flags = []
+        end
+        
+        def resize(width: width, height: height, type: nil)
+          tap do
+            resize_type = type ? "#{ type }-resize" : 'resize'
+            self.flags << "-#{ resize_type } #{ width }x#{ height }\!"
+          end
+        end
+        
+        def adaptive_resize(width: width, height: height)
+          resize width: width, height: height, type: 'adaptive'
+        end
+        
+        def percentage_resize(percentage, type: nil)
+          tap do
+            resize_type = type ? "#{ type }-resize" : 'resize'
+            self.flags << "-#{ resize_type } #{ percentage }% +repage"
+          end
+        end
+        
+        def adaptive_percentage_resize(percentage)
+          percentage_resize percentage, type: 'adaptive'
+        end
+        
+        def quality(percentage)
+          tap do
+            self.flags << "-quality #{ percentage }%"
+          end
+        end
+        
+        def crop(width: width, height: height, top: top, left: left)
+          tap do
+            self.flags << "-crop #{ width }x#{ height }+#{ left }+#{ top } +repage"
+          end
+        end
+        
+        def crop_center(width: width, height: height, top: 0, left: 0)
+          tap do
+            self.flags << "-gravity Center -crop #{ width }x#{ height }+#{ left }+#{ top } +repage"
+          end
+        end
+        
+        def negate
+          tap do
+            self.flags << "-negate"
+          end
+        end
+        alias_method :invert, :negate
+        
+        def write_to(path: nil, prefix: nil, postfix: nil)
+          to(path: path, prefix: prefix, postfix: postfix).write
+        end
+        
+        def to(path: nil, prefix: nil, postfix: nil)
+          raise ImageConversionError.new('Cannot convert an image without an output path') unless path || prefix || postfix
+          tap do
+            if prefix || postfix
+              input_file = input_image.path
+              input_path = File.dirname input_file
+              input_ext = File.extname input_file
+              input_name = File.basename(input_file).sub input_ext, ''
+              
+              input_name = "#{ prefix }_#{ input_name }" if prefix
+              input_name = "#{ input_name }_#{ postfix }" if postfix
+              path = "#{ input_path }/#{ input_name }#{ input_ext }"
+            end
+            
+            self.output_image = Image.new path: path, raise_exceptions: raise_exceptions
+          end
+        end
+        
+        def write
+          raise ImageConversionError.new('Cannot convert an image without an output path') unless output_image
+          
+          output_image.tap do
+            success = system "convert #{ input_image.path } #{ flags.join(' ') } #{ output_image.path }"
+            raise ImageConversionError.new('Image conversion failed') unless success
+            `rm -f '#{ input_image.path }'` if remove_original && input_image.path != output_image.path
+            output_image.optimize if optimize
+          end
+        end
+      end
+      
+      def convert_image(path, remove_original: true, optimize: true)
+        Converter.new path: path, remove_original: remove_original, optimize: optimize
+      end
+      
+      def convert_to_jpeg(path, remove_original: true, optimize: true)
+        _simple_convert path, 'jpg', remove_original: remove_original, optimize: optimize
+      end
+      
+      def convert_to_png(path, remove_original: true, optimize: true)
+        _simple_convert path, 'png', remove_original: remove_original, optimize: optimize
+      end
+      
+      def _simple_convert(path, extension, remove_original: true, optimize: true)
+        out_path = path.sub(/#{ File.extname(path) }$/, ".#{ extension }")
+        convert_image(path, remove_original: remove_original, optimize: optimize).to(path: out_path).write
+      end
+    end
+  end
+end

data/lib/zooniverse_data/helpers/transport.rb

@@ -0,0 +1,67 @@
+require 'timeout'
+require 'aws-sdk'
+
+module ZooniverseData
+  module Helpers
+    module Transport
+      extend ActiveSupport::Concern
+      
+      module ClassMethods
+        def s3
+          @s3 ||= AWS::S3.new
+        end
+      end
+      
+      def download(from: nil, to: nil, timeout: 60)
+        _with_retries(20) do
+          `rm -f '#{ to }'`
+          Timeout::timeout(timeout) do
+            `wget '#{ from }' -t 50 -c -q -O '#{ to }'`
+          end
+          
+          raise 'File not downloaded' unless File.exists?(to)
+          raise 'File is empty' unless File.new(to).size > 0
+        end
+      end
+      
+      def upload(from: nil, to: nil, content_type: nil)
+        content_type ||= `file -bI '#{ from }'`.chomp.split(';').first
+        path = [bucket_path, to].compact.join('/').gsub(/^\//, '').gsub '//', '/'
+        obj = bucket.objects[path]
+        
+        _with_retries(20) do
+          obj.write file: from, acl: :public_read, content_type: content_type
+          raise 'File not uploaded' unless obj.exists?
+        end
+        
+        path
+      end
+      
+      def bucket
+        @bucket ||= self.class.s3.buckets[bucket_name]
+      end
+      
+      def bucket_name
+        @bucket_name ||= manifest.project.bucket
+      end
+      
+      def bucket_path
+        @bucket_path ||= manifest.project.bucket_path
+      end
+      
+      def _with_retries(retries)
+        tries = 0
+        begin
+          yield
+        rescue => e
+          tries += 1
+          if tries < retries
+            retry
+          else
+            raise e
+          end
+        end
+      end
+    end
+  end
+end

data/lib/zooniverse_data/helpers.rb

@@ -0,0 +1,15 @@
+module ZooniverseData
+  module Helpers
+    extend ActiveSupport::Concern
+    
+    included do
+      include Base
+      include Transport
+      include Images
+    end
+    
+    autoload :Base,      'zooniverse_data/helpers/base'
+    autoload :Transport, 'zooniverse_data/helpers/transport'
+    autoload :Images,    'zooniverse_data/helpers/images'
+  end
+end

data/lib/zooniverse_data/projects/default.rb

@@ -0,0 +1,7 @@
+module ZooniverseData
+  module Projects
+    class Default
+      include Helpers
+    end
+  end
+end

data/lib/zooniverse_data/projects/sunspot.rb

@@ -0,0 +1,13 @@
+module ZooniverseData
+  module Projects
+    class Sunspot
+      include Helpers
+      
+      def customize_subject
+        standard = convert_to_jpeg entry.location['standard']
+        inverted = convert_image(standard.path, remove_original: false).invert.write_to(prefix: 'inverted')
+        set_location standard: standard.path, inverted: inverted.path
+      end
+    end
+  end
+end

data/lib/zooniverse_data/projects.rb

@@ -0,0 +1,6 @@
+module ZooniverseData
+  module Projects
+    autoload :Default, 'zooniverse_data/projects/default'
+    autoload :Sunspot, 'zooniverse_data/projects/sunspot'
+  end
+end

data/lib/zooniverse_data/version.rb

@@ -0,0 +1,3 @@
+module ZooniverseData
+  VERSION = '0.0.1'
+end

data/lib/zooniverse_data.rb

@@ -0,0 +1,28 @@
+require 'zooniverse_data/version'
+
+module ZooniverseData
+  class << self
+    attr_accessor :projects
+  end
+  self.projects = { }
+  
+  def self.dispatch(manifest: nil, entry: nil)
+    klass = self.projects[manifest.project_id]
+    
+    unless klass
+      klass_name = manifest.project.name.classify
+      klass = if ZooniverseData::Projects.const_defined?(klass_name)
+        "ZooniverseData::Projects::#{ klass_name }".constantize
+      else
+        ZooniverseData::Projects::Default
+      end
+      
+      self.projects[manifest.project_id] = klass
+    end
+    
+    klass.new.customize manifest: manifest, entry: entry
+  end
+  
+  require 'zooniverse_data/helpers'
+  require 'zooniverse_data/projects'
+end

data/zooniverse_data.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'zooniverse_data/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'zooniverse_data'
+  spec.version       = ZooniverseData::VERSION
+  spec.authors       = ['Michael Parrish']
+  spec.email         = ['michael@zooniverse.org']
+  spec.summary       = 'Zooniverse data library'
+  spec.homepage      = 'https://github.com/zooniverse/zooniverse_data'
+  spec.license       = 'MIT'
+  
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake', '10.1.0'
+  spec.add_runtime_dependency 'fastimage', '1.6.0'
+  spec.required_ruby_version = '>= 2.0.0'
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: zooniverse_data
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Michael Parrish
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+description: 
+email:
+- michael@zooniverse.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/zooniverse_data.rb
+- lib/zooniverse_data/helpers.rb
+- lib/zooniverse_data/helpers/base.rb
+- lib/zooniverse_data/helpers/images.rb
+- lib/zooniverse_data/helpers/transport.rb
+- lib/zooniverse_data/projects.rb
+- lib/zooniverse_data/projects/default.rb
+- lib/zooniverse_data/projects/sunspot.rb
+- lib/zooniverse_data/version.rb
+- zooniverse_data.gemspec
+homepage: https://github.com/zooniverse/zooniverse_data
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Zooniverse data library
+test_files: []