url_scraper 0.0.3

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in url_scraper.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Genii Technologies Pvt Ltd. and Akshay Shinde
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,42 @@
+= Url Scraper
+Its a simple plugin for providing facebook style url scraper. Using url scraper you can convert any simple textarea into a smart textarea which auto detects url entered by users and fetches the content of the webpage and displays it on the page. This gem is built on top of gem 'opengraph' by Michael Bleigh. This gem uses the opengraph tags as well as uses meta, title and img tags (in case if opengraph tags are not present) to fetch information about a page.   
+
+
+== Installation
+
+    gem install url_scraper
+
+paste this line in your application.js file
+
+    //= require jquery.scraper
+
+paste this line in your application.css file          
+
+    *= require scraper
+
+Add following to your routes
+
+    match '/scrape_url' => "UrlScraper#scrape", :via => :post    
+    
+== Using the plugin in a view
+
+Create a text area or text field with id="scrape_url" and that's it.
+
+==  Advanced Usage
+
+    require 'url_scraper'
+    
+    movie = UrlScraper.fetch('http://www.rottentomatoes.com/m/1217700-kick_ass/')
+    
+    movie.title # => 'Kick-Ass'
+    movie.movie? # => true
+    movie.image # => 'http://images.rottentomatoes.com/images/movie/custom/00/1217700.jpg'
+    
+If you try to fetch Open Graph information for a URL that doesn't 
+have any, the <tt>fetch</tt> method will return values for title tag and other meta tags specified on the page.
+
+The UrlScraper::Object that is returned is just a Hash with accessors
+built into it, so you can examine what properties you've retrieved like so:
+
+    movie.keys # => ['type','image','title','url']
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/app/assets/javascripts/jquery.scraper.js

@@ -0,0 +1,107 @@
+$(function(){
+  $('.hidden').hide();
+
+  $('#scrape_url').keyup(function(event){
+    if (event.keyCode == 32){
+      url = linkify($(this).val());
+      if (url != null && url.length > 0)
+        scrapUrl(url.toString());
+    }
+  });
+
+  $('#scrape_url').bind('paste', function () {
+      var url;
+      setTimeout(function () {
+        url = linkify($('#scrape_url').val());
+        if (url != null && url.length > 0)
+          scrapUrl(url.toString());
+      }, 100);
+  });
+
+  function linkify(inputText) {
+    var replacedText, replacePattern1, replacePattern2, replacePattern3;
+
+    //URLs starting with http://, https://, or ftp://
+    replacePattern1 = /((https?|ftp):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/gim;
+    replacedText = inputText.replace(replacePattern1, '$1');
+
+    //URLs starting with "www." (without // before it, or it'd re-link the ones done above).
+    replacePattern2 = /(^|[^\/])(www\.[\S]+(\b|$))/gim;
+    replacedText = replacedText.replace(replacePattern2, 'http://$2');
+
+    url = replacedText.match(replacePattern1);
+
+    // console.log(replacedText);
+    // console.log(url);
+    return(url);
+  }
+
+  function scrapUrl(url){
+    $.ajax({
+      url: "/scrape_url",
+      data: {
+        url: url,
+      },
+      type: 'post',
+      success: function(data){
+        var container = document.createElement('div');
+        container.className = "scraped_content";
+        $('#scrape_url').after(container);
+        
+        console.log(data);
+        
+        //Check if a video is present
+        if(data.video != undefined)
+          alert("video pending");
+        else if(data.image.length > 0)
+        { 
+          image_slider = document.createElement('div');
+          image_slider.setAttribute("id", "image_slider");
+          bjqs_ul = document.createElement('ul');
+          bjqs_ul.className = "bjqs"
+          $(image_slider).append(bjqs_ul);
+          $('.scraped_content').append(image_slider);
+
+          if(data.image instanceof Array) {
+            for(image in data.image){
+              if(data.image[image].match("//") == undefined) 
+                $(".scraped_content #image_slider ul.bjqs").append("<li><img src=" + url + data.image[image] + "><img></li>");
+              else
+                $(".scraped_content #image_slider ul.bjqs").append("<li><img src=" + data.image[image] + "><img></li>");
+            }
+          } else if(data.image != null) {
+              if(data.image.match("//") == undefined) 
+                $(".scraped_content #image_slider ul.bjqs").append("<li><img src=" + url + data.image + "><img></li>");
+              else
+                $(".scraped_content #image_slider ul.bjqs").append("<li><img src=" + data.image + "><img></li>");
+          }
+          init_slider();
+        }
+
+        //Add the title container
+        title = document.createElement('input');
+        title.setAttribute("type", "text");
+        title.setAttribute("value", data.title);                
+        $('.scraped_content').append(title);
+
+        //Add the description container
+        description = document.createElement('textarea');
+        if(data.description.length > 0)
+          $(description).append(data.description);
+        else
+          description.setAttribute("placeholder", "This webpage doesn't provide any description. Go ahead and write your own.");                
+        $('.scraped_content').append(description);
+      }
+    });
+  }
+
+  function init_slider(){
+    $('.scraped_content #image_slider').bjqs({
+      'height' : 100,
+      'width' : 170,
+      'responsive' : true,
+      'showmarkers' : false,
+    });
+  };
+
+});

data/app/controllers/url_scraper_controller.rb

@@ -0,0 +1,6 @@
+roclass UrlScraperController < ApplicationController
+  def scrape
+    object = UrlScraper.fetch(params[:url])
+    render :json => object
+  end
+end

data/lib/url_scraper.rb

@@ -0,0 +1,106 @@
+require "url_scraper/version"
+require 'hashie'
+require 'nokogiri'
+require 'restclient'
+require 'logger'
+require 'thor'
+
+module UrlScraper
+  # Tell rails to load all assets
+  class Engine < Rails::Engine
+  
+  end
+
+  class CLI < Thor
+
+  end
+  
+  # Handles the url request
+
+  # Fetch Open Graph data from the specified URI. Makes an
+  # HTTP GET request and returns an UrlScraper::Object if there
+  # is data to be found or <tt>false</tt> if there isn't.
+  #
+  # Pass <tt>false</tt> for the second argument if you want to
+  # see invalid (i.e. missing a required attribute) data.
+  
+  def self.fetch(uri, strict = true)  
+    parse(RestClient.get(uri).body, strict)
+    rescue RestClient::Exception, SocketError
+      false
+  end
+  
+  def self.parse(html, strict = true)
+    logger = Logger.new(STDOUT)
+    doc = Nokogiri::HTML.parse(html)
+    page = UrlScraper::Object.new
+    doc.css('meta').each do |m|
+      if m.attribute('property') && m.attribute('property').to_s.match(/^og:(.+)$/i)
+        page[$1.gsub('-','_')] = m.attribute('content').to_s
+      end
+    end
+
+    page.title = (doc.at_css('title').text rescue nil) if page.title.nil? 
+    if page.description.nil?
+      page.description = doc.at_css("meta[name='description']")['content'] unless doc.at_css("meta[name='description']").nil?
+    end
+    if page.image.nil?
+      image_array = doc.css("img").take(3).collect{|img| img['src']}
+      page.image = image_array unless image_array.empty?
+    end
+    # return false if page.keys.empty?
+    # return false unless page.valid? if strict
+    page
+    # return doc
+  end
+  
+  TYPES = {
+    'activity' => %w(activity sport),
+    'business' => %w(bar company cafe hotel restaurant),
+    'group' => %w(cause sports_league sports_team),
+    'organization' => %w(band government non_profit school university),
+    'person' => %w(actor athlete author director musician politician public_figure),
+    'place' => %w(city country landmark state_province),
+    'product' => %w(album book drink food game movie product song tv_show),
+    'website' => %w(blog website)
+  }
+  
+  # The UrlScraper::Object is a Hash with method accessors for
+  # all detected Open Graph attributes.
+  class Object < Hashie::Mash
+    MANDATORY_ATTRIBUTES = %w(title type image url)
+    
+    # The object type.
+    def type
+      self['type']
+    end
+    
+    # The schema under which this particular object lies. May be any of
+    # the keys of the TYPES constant.
+    def schema
+      UrlScraper::TYPES.each_pair do |schema, types| 
+        return schema if types.include?(self.type)
+      end
+      nil
+    end
+    
+    UrlScraper::TYPES.values.flatten.each do |type|
+      define_method "#{type}?" do
+        self.type == type
+      end
+    end
+    
+    UrlScraper::TYPES.keys.each do |scheme|
+      define_method "#{scheme}?" do
+        self.type == scheme || UrlScraper::TYPES[scheme].include?(self.type)
+      end
+    end
+    
+    # If the Open Graph information for this object doesn't contain
+    # the mandatory attributes, this will be <tt>false</tt>.
+    def valid?
+      MANDATORY_ATTRIBUTES.each{|a| return false unless self[a]}
+      true
+    end
+  end
+end

data/lib/url_scraper/version.rb

@@ -0,0 +1,3 @@
+module UrlScraper
+  VERSION = "0.0.3"
+end

data/url_scraper.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'url_scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "url_scraper"
+  spec.version       = UrlScraper::VERSION
+  spec.authors       = ["Super Engineer"]
+  spec.email         = ["akshayshinde7@gmail.com"]
+  spec.description   = %q{A simple plugin for extracting information from url entered by user (Something
+  like what facebook does). This gem is built on top of opengraph gem created by michael
+  bleigh.}
+  spec.summary       = %q{A simple plugin for extracting information from url entered by user (Something
+  like what facebook does). This gem is built on top of opengraph gem created by michael
+  bleigh.}
+  spec.homepage      = "http://github.com/super-engineer/url_scraper"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency 'hashie'
+  spec.add_dependency 'nokogiri', '~> 1.5.9'
+  spec.add_dependency 'rest-client', '~> 1.6.7'
+  spec.add_dependency 'thor'
+end

metadata

@@ -0,0 +1,157 @@
+--- !ruby/object:Gem::Specification
+name: url_scraper
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+  prerelease: 
+platform: ruby
+authors:
+- Super Engineer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.9
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "A simple plugin for extracting information from url entered by user
+  (Something\n  like what facebook does). This gem is built on top of opengraph gem
+  created by michael\n  bleigh."
+email:
+- akshayshinde7@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- app/assets/javascripts/jquery.scraper.js
+- app/assets/stylesheets/scraper.css
+- app/controllers/url_scraper_controller.rb
+- lib/url_scraper.rb
+- lib/url_scraper/version.rb
+- url_scraper.gemspec
+homepage: http://github.com/super-engineer/url_scraper
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A simple plugin for extracting information from url entered by user (Something
+  like what facebook does). This gem is built on top of opengraph gem created by michael
+  bleigh.
+test_files: []