image_metadata_scraper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bd0aa9cda13195b2788ff0f711d86872401ec746
+  data.tar.gz: 971691ea282517cfce83ad72277543e0cd3b6c69
+SHA512:
+  metadata.gz: 34f9410fedb0716de1bb3bf9ffa659f4a9753d405777d740b247bedcf8dbf7cba206f81c6dc4e8efed5197007df5fd0a59bbfc76c19b64279623a900005c5f67
+  data.tar.gz: 4ce3f666c1fa9e4da10c76f4b74834fe346453d92419a8bd4ad7cdea9a6f67042e6e1509c9ece9938435747d71e8895875818d885520a5fda92d3ae7b1b4b1cd

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,6 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.4.0
+after_success:
+  - bundle exec codeclimate-test-reporter

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,116 @@
+CC0 1.0 Universal
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific
+works ("Commons") that the public can reliably and without fear of later
+claims of infringement build upon, modify, incorporate in other works, reuse
+and redistribute as freely as possible in any form whatsoever and for any
+purposes, including without limitation commercial purposes. These owners may
+contribute to the Commons to promote the ideal of a free culture and the
+further production of creative, cultural and scientific works, or to gain
+reputation or greater distribution for their Work in part through the use and
+efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation
+of additional consideration or compensation, the person associating CC0 with a
+Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
+and publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not limited
+to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display, communicate,
+  and translate a Work;
+
+  ii. moral rights retained by the original author(s) and/or performer(s);
+
+  iii. publicity and privacy rights pertaining to a person's image or likeness
+  depicted in a Work;
+
+  iv. rights protecting against unfair competition in regards to a Work,
+  subject to the limitations in paragraph 4(a), below;
+
+  v. rights protecting the extraction, dissemination, use and reuse of data in
+  a Work;
+
+  vi. database rights (such as those arising under Directive 96/9/EC of the
+  European Parliament and of the Council of 11 March 1996 on the legal
+  protection of databases, and under any national implementation thereof,
+  including any amended or successor version of such directive); and
+
+  vii. other similar, equivalent or corresponding rights throughout the world
+  based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
+and Related Rights and associated claims and causes of action, whether now
+known or unknown (including existing as well as future claims and causes of
+action), in the Work (i) in all territories worldwide, (ii) for the maximum
+duration provided by applicable law or treaty (including future time
+extensions), (iii) in any current or future medium and for any number of
+copies, and (iv) for any purpose whatsoever, including without limitation
+commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
+the Waiver for the benefit of each member of the public at large and to the
+detriment of Affirmer's heirs and successors, fully intending that such Waiver
+shall not be subject to revocation, rescission, cancellation, termination, or
+any other legal or equitable action to disrupt the quiet enjoyment of the Work
+by the public as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free,
+non transferable, non sublicensable, non exclusive, irrevocable and
+unconditional license to exercise Affirmer's Copyright and Related Rights in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "License"). The License shall be deemed effective as
+of the date CC0 was applied by Affirmer to the Work. Should any part of the
+License for any reason be judged legally invalid or ineffective under
+applicable law, such partial invalidity or ineffectiveness shall not
+invalidate the remainder of the License, and in such case Affirmer hereby
+affirms that he or she will not (i) exercise any of his or her remaining
+Copyright and Related Rights in the Work or (ii) assert any associated claims
+and causes of action with respect to the Work, in either case contrary to
+Affirmer's express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+  a. No trademark or patent rights held by Affirmer are waived, abandoned,
+  surrendered, licensed or otherwise affected by this document.
+
+  b. Affirmer offers the Work as-is and makes no representations or warranties
+  of any kind concerning the Work, express, implied, statutory or otherwise,
+  including without limitation warranties of title, merchantability, fitness
+  for a particular purpose, non infringement, or the absence of latent or
+  other defects, accuracy, or the present or absence of errors, whether or not
+  discoverable, all to the greatest extent permissible under applicable law.
+
+  c. Affirmer disclaims responsibility for clearing rights of other persons
+  that may apply to the Work or any use thereof, including without limitation
+  any person's Copyright and Related Rights in the Work. Further, Affirmer
+  disclaims responsibility for obtaining any necessary consents, permissions
+  or other rights required for any use of the Work.
+
+  d. Affirmer understands and acknowledges that Creative Commons is not a
+  party to this document and has no duty or obligation with respect to this
+  CC0 or use of the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>

data/README.md

@@ -0,0 +1,13 @@
+# ImageMetadataScraper
+
+```ruby
+require 'image_metadata_scraper'
+ImageMetadataScraper.scrape 'http://fav.me/shorturl'
+
+=> { :image_url => "http://orig.deviantart.net/image-file-url.jpg",
+     :thumbnail_url => "http://pre.deviantart.net/small-image-version-url.jpg",
+     :artist => "artist",
+     :url => "http://artist.deviantart.com/art/some-piece-123" }
+```
+
+[Supported providers](https://github.com/little-bobby-tables/image_metadata_scraper/blob/master/lib/image_metadata_scraper.rb#L11)

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task default: :test

data/image_metadata_scraper.gemspec

@@ -0,0 +1,28 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+
+require 'image_metadata_scraper/version'
+Gem::Specification.new do |spec|
+  spec.name          = 'image_metadata_scraper'
+  spec.version       = ImageMetadataScraper::VERSION
+  spec.authors       = ['little-bobby-tables']
+  spec.email         = ['little-bobby-tables@users.noreply.github.com']
+
+  spec.summary       = 'Simple image metadata scraper.'
+  spec.homepage      = 'https://github.com/little-bobby-tables/image_metadata_scraper'
+  spec.license       = 'CC0-1.0'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'activesupport'
+
+  spec.add_development_dependency 'openssl'
+  spec.add_development_dependency 'vcr'
+  spec.add_development_dependency 'webmock'
+  spec.add_development_dependency 'minitest-reporters'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+end

data/lib/image_metadata_scraper/deviantart.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require 'nokogiri'
+require 'net/http'
+
+module ImageMetadataScraper
+  module DeviantArt
+    def self.post(url)
+      response = URI(url).try do |uri|
+        Net::HTTP.new(uri.host).get(uri.request_uri, { 'Cookie' => 'agegate_state=1' })
+      end
+      page = Nokogiri::HTML(response.body)
+
+      # Image file URL: downloads enabled
+      image_file_url = page.at('a.dev-page-download')&.attr('href')
+      image_file_url &&= _follow_download_url(image_file_url, response)
+
+      # Image file URL: downloads disabled
+      image_file_url ||= page.at('.dev-view-deviation img.dev-content-full').attr('src')
+
+      thumbnail_url = page.at('.dev-view-deviation img.dev-content-normal').attr('src')
+
+      artist_name = page.at('.dev-title-container .username').content
+
+      canonical_page_url = page.at('meta[property="og:url"]')&.attr('content')
+
+      { image_url: image_file_url, thumbnail_url: thumbnail_url, artist: artist_name, url: canonical_page_url }
+    end
+
+    def self.direct_link(url)
+      # Direct links have the fav.me shortcode as a part of the URL, we just need to grab that.
+      shortcode = url.match(/-(.+)\..+\z/)[1]
+      url = "http://fav.me/#{shortcode}"
+      post(ImageMetadataScraper.redirect_from(url))
+    end
+
+    def self._follow_download_url(url, previous_request)
+      cookies = previous_request['set-cookie'].split(';').first
+      uri = URI(url)
+      Net::HTTP.new(uri.host).head(uri.request_uri, { 'Cookie' => cookies })['location']
+    end
+  end
+end

data/lib/image_metadata_scraper/tumblr.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require 'nokogiri'
+require 'open-uri'
+
+module ImageMetadataScraper
+  module Tumblr
+    def self.post(url)
+      api_url = url.match /\A(?<blog>https?:\/\/.*tumblr.com)\/(post|image)\/(?<post_id>\d+)/ do |m|
+        "#{m[:blog]}/api/read?id=#{m[:post_id]}"
+      end
+      xml = Nokogiri::XML(open(api_url))
+
+      image_file_url = xml.at('//photo-url[@max-width="1280"]').content
+
+      thumbnail_url = xml.at('//photo-url[@max-width="500"]').content
+
+      artist_name = xml.at('//tumblelog').attr('name')
+
+      canonical_page_url = xml.at('//post').attr('url-with-slug')
+
+      { image_url: image_file_url, thumbnail_url: thumbnail_url, artist: artist_name, url: canonical_page_url }
+    end
+  end
+end

data/lib/image_metadata_scraper/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module ImageMetadataScraper
+  VERSION = '0.1.0'
+end

data/lib/image_metadata_scraper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+require 'net/http'
+
+require 'image_metadata_scraper/version'
+require 'image_metadata_scraper/deviantart'
+require 'image_metadata_scraper/tumblr'
+
+module ImageMetadataScraper
+  IMAGE_FILE_URL = /\Ahttps?:\/\/.*\.(jpg|jpeg|png|gif|svg)/
+
+  SCRAPERS = {
+    /\Ahttps?:\/\/.+deviantart\.com\/.+/ => DeviantArt.method(:post),
+    /\Ahttps?:\/\/.+deviantart\.net\/.+d.+/ => DeviantArt.method(:direct_link),
+    /\Ahttps?:\/\/.+tumblr\.com\/(post|image)\/.+/ => Tumblr.method(:post),
+    IMAGE_FILE_URL => ->(url) { { image_url: url, thumbnail_url: url } }
+  }.freeze
+
+  # Returns a hash of scraped image metadata that always contains:
+  # +image_url+:     URL to the largest available image file
+  # +thumbnail_url+: URL to a small version of the image
+  #
+  # and includes, if applicable,
+  # +artist+:    the name of the artist (blogger)
+  # +url+:       canonical URL to the image page (e.g. DeviantArt post)
+  #
+  # Returns nil if scraping fails.
+  def self.scrape(url)
+    url = http_url(url) or return
+    url = redirect_from(url)
+
+    scraper = SCRAPERS.detect { |regex, _| url =~ regex }&.last or return
+    scraper.call(url)
+  end
+
+  def self.redirect_from(url)
+    response = Net::HTTP.get_response(URI(url.strip))
+    case response.code when '301', '302'
+      response.header['location']
+    else
+      url
+    end
+  end
+
+  def self.http_url(url)
+    return if url.blank?
+
+    url = url.strip
+    scheme = url.match(/\A.+:\/\//)
+
+    if scheme.nil?
+      "http://#{url}"
+    elsif scheme.to_s == 'http://' || scheme.to_s == 'https://'
+      url
+    end
+  end
+end

metadata

@@ -0,0 +1,167 @@
+--- !ruby/object:Gem::Specification
+name: image_metadata_scraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- little-bobby-tables
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: openssl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-reporters
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- little-bobby-tables@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- image_metadata_scraper.gemspec
+- lib/image_metadata_scraper.rb
+- lib/image_metadata_scraper/deviantart.rb
+- lib/image_metadata_scraper/tumblr.rb
+- lib/image_metadata_scraper/version.rb
+homepage: https://github.com/little-bobby-tables/image_metadata_scraper
+licenses:
+- CC0-1.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: Simple image metadata scraper.
+test_files: []