rinkusukurepa 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ed57cee41ec46988d39ebdfe08c1e9baa1a4478
+  data.tar.gz: 1607ef131fc9aa0baba9a89cf655cb00e783e5ba
+SHA512:
+  metadata.gz: 481fa1ccdae83c4fb40d1314b97095166bffae554a3bb0eb77dc6c9f5e3d6b04dbaa727f1ac1122d8a3d5ba8926801664582680bd77b035266a84ece784ccc9b
+  data.tar.gz: d39f067f5b389aec2b2e3421d0a2f0b98ca8339df66bda75aa9a9db7a8804c34160278eb7fff6132724fa653da28c9fa6d2e3136aa09e5c962fc0ab3867d79ec

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.2
+before_install: gem install bundler -v 1.10.3

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rinkusukurepa.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 rewin0087
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# Rinkusukurepa
+
+A library for Scraping a webpage by it's url and return the web page title, description, site name, images, favicon and video (if there's a video). Inspired by facebook url sharer.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rinkusukurepa'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rinkusukurepa
+
+## Usage
+  web_page = Rinkusukurepa.parse_url!('https://github.com/rewin0087/rinkusukurepa')
+  web_page.title # return the web page title
+  web_page.description # return the web page description
+  web_page.images # return all the web page images
+  web_page.site_name # return site name
+  web_page.video # return video if present
+  web_page.page_type # return page type
+  web_page.page_document # return raw nokigiri document object
+  web_page.attributes # return a hash with icon, title, description, images, site_name, video and page_type
+
+  Scrape and get specific attribute
+
+  web_page = Rinkusukurepa.new('https://github.com/rewin0087/rinkusukurepa')
+  web_page.parse_url # it will scrape and return the document of the url
+  web_page.get_icon  # it will find the icon from the document and return the icon
+  web_page.get_title # it will find the title from the document and return the title
+  web_page.get_description # it will find the description from the document and return the description
+  web_page.get_images # it will find the images from the document and return the images
+  web_page.get_site_name # it will find the site name from the document and return the site name
+  web_page.get_video # it will find the video from the document and return the video
+  web_page.get_page_type # it will find the page type from the document and return the page type
+
+  To customize some of the configurations
+  create a file in the config/initializers/rinkusukurepa.rb
+  and put:
+  Rinkusukurepa.configure do |config|
+    config.image_min_width = 200 # default 150
+    config.image_min_height = 200 # default 100
+    config.max_image = 20 # default 20
+    config.page_types = ['article', 'post'] # default ['website', 'video', 'sound']
+    config.image_extensions = /(.png|.jpg)/ # default /(.png|PNG|.jpg|JPG|.jpeg|JPEG|BMP|.bmp|.gif|GIF)/
+  end
+
+## Contributing
+
+  1. Fork it
+  2. Create your feature branch (git checkout -b my-new-feature)
+  3. Commit your changes (git commit -am 'Add some feature')
+  4. Push to the branch (git push origin my-new-feature)
+  5. Create new Pull Request
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rinkusukurepa"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/rinkusukurepa/configuration.rb

@@ -0,0 +1,33 @@
+module Rinkusukurepa
+  module Configuration
+    attr_accessor :image_min_width,
+      :image_min_height,
+      :max_image,
+      :page_types,
+      :image_extensions
+
+    def configure
+      yield self
+    end
+
+    def image_min_width
+      @image_min_width || 150
+    end
+
+    def image_min_height
+      @image_min_height || 100
+    end
+
+    def max_image
+      @max_image || 10
+    end
+
+    def page_types
+      @page_types || %w(website video sound)
+    end
+
+    def image_extensions
+      @image_extensions || /(.png|PNG|.jpg|JPG|.jpeg|JPEG|BMP|.bmp|.gif|GIF)/
+    end
+  end
+end

data/lib/rinkusukurepa/scraper.rb

@@ -0,0 +1,239 @@
+require 'uri'
+require 'open-uri'
+require 'open_uri_redirections'
+require 'nokogiri'
+require 'fastimage'
+
+module Rinkusukurepa
+  extend Rinkusukurepa::Configuration
+
+  class Scraper
+    attr_accessor :page_url,
+      :icon,
+      :title,
+      :description,
+      :images,
+      :site_name,
+      :video,
+      :page_type,
+      :page_document
+
+    Rinkusukurepa.page_types.each do |type|
+      define_method "#{type.downcase.strip.gsub(/[^0-9A-Za-z]/, '')}?" do
+        self.type == type
+      end
+
+      define_singleton_method "#{type.downcase.strip.gsub(/[^0-9A-Za-z]/, '')}" do
+        type
+      end
+    end
+
+    def self.parse_url!(url)
+      scraper = Rinkusukurepa::Scraper.new(url)
+      scraper.parse_url!
+      scraper
+    end
+
+    def initialize(url)
+      @page_url = URI.parse(URI.encode(url))
+      @images = []
+    end
+
+    def parse_url
+      @page_document = Nokogiri::HTML(open(@page_url.to_s, :allow_redirections => :all))
+      self
+    end
+
+    def parse_url!
+      parse_url
+      get_icon
+      get_title
+      get_description
+      get_images
+      get_site_name
+      get_video
+      get_page_type
+    end
+
+    def attributes
+      {
+        icon: @icon,
+        title: @title,
+        description: @description,
+        images: @images || [],
+        site_name: @site_name,
+        video: @video,
+        page_type: @page_type
+      }
+    end
+
+    def get_icon
+      icon = @page_document.css('link[rel="icon"]')
+      icon = @page_document.css('link[rel="shortcut icon"]') if icon.nil? || icon.empty?
+
+      if !icon.nil? && !icon.empty?
+        href = icon.first.attributes['href'].value
+        uri = URI.parse(URI.encode(href))
+        if uri.host.nil?
+          @icon = "http://#{@page_url.host}#{uri.path}"
+        else
+          @icon = "http://#{uri.host}#{uri.path}"
+        end
+      end
+
+      @icon
+    end
+
+    def get_title
+      # title
+      title = @page_document.css('meta[property="og:title"]')
+      if title.nil? || title.empty?
+        @title = @page_document.css('title').text
+      else
+        # parse from meta
+        @title = title.first.attributes['content'].value
+      end
+
+      @title
+    end
+
+    def get_description
+      # Description
+      description = @page_document.css('meta[property="og:description"]')
+      if description.nil? || description.empty?
+        description = @page_document.css('meta[name="description"]')
+        if !description.nil? && !description.empty?
+          @description = description.first.attributes['content'].value
+        end
+      else
+        # parse from meta
+        @description = description.first.attributes['content'].value
+      end
+
+      @description
+    end
+
+    def get_images
+      # check if url is an image
+      if File.extname(@page_url.to_s).match(Rinkusukurepa.image_extensions)
+        @images << @page_url.to_s
+        return @images
+      end
+
+      # Image Preview
+      preview = @page_document.css('meta[property="og:image"]')
+      if preview.nil? || preview.empty?
+        retrieve_all_images_from_document
+      else
+        # parse from meta
+        @images = preview.map do |p|
+          url = p.attributes['content'].value
+          uri = URI.parse(URI.encode(url.strip! || url))
+          if uri.to_s.match(/http/)
+            if uri.host.nil?
+              "http://#{@page_url.host}#{uri.to_s}"
+            else
+              uri.to_s
+            end
+            if http_image = FastImage.size(uri.to_s)
+              if http_image[0] > Rinkusukurepa.image_min_width && http_image[1] > Rinkusukurepa.image_min_height
+                uri.to_s
+              end
+            end
+          end
+        end.compact.uniq
+
+        if @images.nil? || @images.empty?
+          retrieve_all_images_from_document
+        end
+      end
+
+      @images
+    end
+
+    def get_site_name
+      # Site Name
+      site_name = @page_document.css('meta[property="og:site_name"]')
+      if site_name.nil? || site_name.empty?
+        @site_name = @page_url.host
+      else
+        # parse from meta
+        @site_name = site_name.first.attributes['content'].value
+      end
+
+      @site_name
+    end
+
+    def get_video
+      # Video
+      videos = @page_document.css('meta[property="og:video:secure_url"]')
+      if !videos.nil? && !videos.empty?
+        # parse from meta
+        @video = videos.find { |v| v.attributes['content'].value.include?('embed') }.attributes['content'].value
+      end
+
+      @video
+    end
+
+    def get_page_type
+      # Type
+      type = @page_document.css('meta[property="og:type"]')
+      if type.nil? || type.empty?
+        @page_type = Rinkusukurepa::Scraper.website
+      else
+        # parse from meta
+        page_type = type.first.attributes['content'].value
+        @page_type = Rinkusukurepa.page_types.include?(page_type) ? page_type : Rinkusukurepa::Scraper.website
+      end
+
+      @page_type
+    end
+
+    protected
+
+    def retrieve_all_images_from_document
+      # get all img
+      preview = @page_document.css('img')
+      if !preview.nil? && !preview.empty?
+        previews = preview[0..Rinkusukurepa.max_image].map do |p|
+          image = p.attributes['src']
+          unless image.blank?
+            url = nil
+            if image.value.match(Rinkusukurepa.image_extensions) && image.value.match(/http/)
+               uri = URI.parse(URI.encode(image.value.strip! || image.value))
+              if uri.host.nil?
+                url = "http://#{@page_url.host}/#{uri.to_s}"
+              else
+                url = uri.to_s
+              end
+            end
+
+            unless url.nil?
+              # NOTE: fallback mini magick if fast image return's nil
+              # fast image
+              if http_image = ::FastImage.size(url)
+                if http_image[0] > Rinkusukurepa.image_min_width && http_image[1] > Rinkusukurepa.image_min_height
+                  url
+                end
+              # mini magick
+              else
+                begin
+                  http_image = ::MiniMagick::Image.open(url)
+                  if http_image.present?
+                    if http_image.width > Rinkusukurepa.image_min_width && http_image.height > Rinkusukurepa.image_min_height
+                      url
+                    end
+                  end
+                rescue => e
+                  nil
+                end
+              end
+            end
+          end
+        end.compact.uniq
+        # set scraped images
+        @images = previews
+      end
+    end
+  end
+end

data/lib/rinkusukurepa/version.rb

@@ -0,0 +1,3 @@
+module Rinkusukurepa
+  VERSION = "0.1.0"
+end

data/lib/rinkusukurepa.rb

@@ -0,0 +1,19 @@
+require "rinkusukurepa/version"
+require "rinkusukurepa/configuration"
+require "rinkusukurepa/scraper"
+
+module Rinkusukurepa
+  class << self
+    def new(url)
+      scraper.new(url)
+    end
+
+    def scraper
+      Rinkusukurepa::Scraper
+    end
+
+    def parse_url!(url)
+      Rinkusukurepa::Scraper.parse_url!(url)
+    end
+  end
+end

data/rinkusukurepa.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rinkusukurepa/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rinkusukurepa"
+  spec.version       = Rinkusukurepa::VERSION
+  spec.authors       = ["rewin0087"]
+  spec.email         = ["erwin_rulezzz87@yahoo.com"]
+
+  spec.summary       = %q{A library for Scraping a webpage by it's url and return the web page title, description, site name, images, favicon and video (if there's a video). Inspired by facebook url sharer.}
+  spec.description   = %q{A library for Scraping a webpage by it's url and return the web page title, description, site name, images, favicon and video (if there's a video). Inspired by facebook url sharer. }
+  spec.homepage      = "https://github.com/rewin0087/rinkusukurepa"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+
+  spec.add_runtime_dependency "nokogiri", "~> 1.6.7"
+  spec.add_runtime_dependency "open_uri_redirections", "~> 0.2.1"
+  spec.add_runtime_dependency "fastimage", "~> 1.8.1"
+end

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification
+name: rinkusukurepa
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- rewin0087
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+- !ruby/object:Gem::Dependency
+  name: open_uri_redirections
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+description: 'A library for Scraping a webpage by it''s url and return the web page
+  title, description, site name, images, favicon and video (if there''s a video).
+  Inspired by facebook url sharer. '
+email:
+- erwin_rulezzz87@yahoo.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/rinkusukurepa.rb
+- lib/rinkusukurepa/configuration.rb
+- lib/rinkusukurepa/scraper.rb
+- lib/rinkusukurepa/version.rb
+- rinkusukurepa.gemspec
+homepage: https://github.com/rewin0087/rinkusukurepa
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: A library for Scraping a webpage by it's url and return the web page title,
+  description, site name, images, favicon and video (if there's a video). Inspired
+  by facebook url sharer.
+test_files: []