links_processor 0.5.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in links_processor.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/links_processor.rb

@@ -0,0 +1,190 @@
+# To change this template, choose Tools | Templates
+# and open the template in the editor.
+require 'net/http'
+require 'nokogiri'
+require 'open-uri'
+require 'oembed'
+require 'image_size'
+
+module LinksProcessor
+  class LinksProcessor
+  
+    def initialize(type, source = nil)
+      @type = type
+      source &&= source.capitalize
+      @processor = OEmbed::Providers.const_get(source) if source && OEmbed::Providers.constants.include?(source)
+      @source = source
+    end
+  
+    def process
+      self.send :"process_#{@type}"
+    end
+ 
+    def process_video(url, video_options)
+      video_options.merge!(:format => :json)
+      processed_video = @processor.get(url, video_options)
+      processed_video.uri = URI.parse processed_video.request_url
+      processed_video.sld = get_sld processed_video.uri.host
+      if @source == 'Youtube'
+        doc = Nokogiri::HTML::fragment processed_video.html
+        if doc.at_xpath('.//param[@name=\'movie\']')
+          doc.at_xpath('.//param[@name=\'movie\']')['value'] += '&autoplay=1'
+          doc.at_xpath('.//embed')['src'] += '&autoplay=1'
+        elsif doc.at_xpath('.//iframe')
+          doc.at_xpath('.//iframe')['src'] += '&autoplay=1'
+        end
+        processed_video.fields['html'] = doc.to_html
+      end
+      processed_video
+    end
+  
+    def process_doc(url, doc_options)
+      doc_options.merge!(:format => :json)
+      processed_doc =  @processor.get(url, doc_options)
+      processed_doc.uri = URI.parse processed_doc.request_url
+      processed_doc.sld = get_sld processed_doc.uri.host
+      doc = Nokogiri::HTML::fragment processed_doc.html
+      if @source == 'Scribd'
+        processed_doc.height = doc.at_xpath('.//object')['height']
+      elsif @source == 'Slideshare'
+        processed_doc.fields['html'] =  doc.at_xpath('.//iframe').to_s
+      end
+      processed_doc
+    end
+  
+    MAX_IMAGESIZE = '512K'
+    MIN_IMAGESIZE = '1K'
+    MAX_IMAGES = 10
+    ASPECT_RATIO_RANGE = 0.25..4
+  
+    def process_link(url, link_options = nil)
+      uri = URI.parse url
+      uri_sld = get_sld uri.host
+      data = {:images => [], :title => nil, :description => nil}
+      doc = Nokogiri::HTML(open(uri.to_s))
+    
+      data[:title] ||= (doc.at_xpath('/html/head/meta[@property=\'og:title\']')['content'] rescue nil)
+      data[:title] ||= (doc.at_xpath('/html/head/meta[@name=\'title\']')['content'] rescue nil)
+      data[:title] ||= (doc.xpath('/html/head/title').text rescue nil)
+      data[:description] = doc.at_xpath('/html/head/meta[@name=\'description\']')['content'] rescue nil
+      data[:sld] = uri_sld
+      data[:uri] = uri
+      image = doc.at_xpath('/html/head/meta[@property=\'og:image\']')['content'] rescue nil
+      if image
+        data[:images] << image
+      else
+        images =  doc.xpath('/html/body//img')
+        unsized_images = []
+        
+        # filter out unsized images
+        images = images.select do |img|
+          width,height = ['width','height'].map{|x| img[x] && img[x].chomp('px')}
+          valid_size = width && height && width =~ /\d+/ && height =~ /\d+/
+          unsized_images << img unless valid_size
+          valid_size
+        end
+        
+        if images.empty?
+          images += update_images_sizes(unsized_images, uri)
+        end
+        
+        images.each do |img|
+          break if data[:images].count > MAX_IMAGES
+          img_uri = URI.parse img['src'] rescue nil
+          
+          # Skip if invalid image src or dimensions
+          next unless img_uri
+        
+          width, height = img['width'].to_i, img['height'].to_i
+        
+          # Skip if ads images
+          next if img_uri.to_s =~ /(ad|ads)\./i
+        
+          # Skip if not matching image constraints
+          next if width == 0 || height == 0 || width < 50 || height < 50 || ASPECT_RATIO_RANGE.exclude?(width/height)
+          
+          # make image url host and protocol same as page if the src is not a complete url
+          img_uri.path = uri.path if img_uri.relative? && !uri.host
+          img_uri.scheme = uri.scheme unless img_uri.scheme
+          img_uri.host = uri.host unless img_uri.host
+          
+          img_sld = get_sld img_uri.host
+        
+          # Skip if second level domain of images is not equal to page second level domain example static.example.com and www.example.com match
+          # as both have sld example.com
+          next unless img_sld == uri_sld unless img_sld
+          
+          data[:images] << img_uri.to_s
+        end
+      end
+      data
+    end
+    
+    def update_images_sizes(images, parent_uri)
+      sized_images = []
+      images.select do |img|
+        break if sized_images.size > 4
+        img_uri = URI.parse(img['src']) rescue nil
+        next unless img_uri
+        # Try to find if filename itself ends with size format such as 100x35 .. else a request is made
+        if match = /(\d{1,4})(x|X)(\d{1,4})$/.match(File.basename(img_uri.path, '.*'))
+           img['width'] = match[1]
+           img['height'] =  match[3]
+        else 
+          img_uri.scheme = parent_uri.scheme unless img_uri.scheme
+          img_uri.host = parent_uri.host unless img_uri.host
+          open(img_uri.to_s, "rb") do |fh|
+            img_size = ImageSize.new(fh.read)
+            img['width'] = img_size.get_width.to_s unless img_size.get_width.nil?
+            img['height'] =  img_size.get_height.to_s unless img_size.get_height.nil?
+          end
+        end
+        sized_images << img if img['width'] && img['height']
+      end
+      sized_images
+    end
+  
+    def validate_size
+      response = nil
+      Net::HTTP.start('www.biostat.wisc.edu', 80) {|http|
+        response = http.head('/bcg/categories/languages/ruby/ruby_logo.png')
+      }
+      p response['content-length']
+    end
+  
+    private
+    
+    def open_url
+      
+    end
+    
+    def get_sld(domain)
+      splits = domain.split('.')
+      splits.count > 2 ? splits[-2..-1].join('.') : domain
+    end
+  end
+
+  # Monkey Patches for OEMBED && OPEN-URI
+  module OEmbed::Formatter::JSON
+    private
+    def self.test_value
+    <<-JSON
+  {"version":1.0, "string":"test", "int":42,"html":"<i>Cool's</i>\\n the \\"word\\"\\u0021"}
+    JSON
+    end
+  end
+
+  class OEmbed::Response
+    attr_accessor :uri, :sld, :height
+  end
+  
+  # Monkey patch OPENURI to follow redirect URLS
+  def OpenURI.redirectable?(uri1, uri2) # :nodoc:
+    # This test is intended to forbid a redirection from http://... to
+    # file:///etc/passwd.
+    # However this is ad hoc.  It should be extensible/configurable.
+    uri1.scheme.downcase == uri2.scheme.downcase ||
+    (/\A(?:http|ftp)\z/i =~ uri1.scheme && /\A(?:https?|ftp)\z/i =~ uri2.scheme)
+  end
+
+end

data/lib/links_processor/version.rb

@@ -0,0 +1,3 @@
+module LinksProcessor
+  VERSION = "0.5.1"
+end

data/links_processor.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "links_processor/version"
+
+Gem::Specification.new do |s|
+  s.name        = "links_processor"
+  s.version     = LinksProcessor::VERSION
+  s.authors     = ["Shadab Ahmed"]
+  s.email       = ["shadab.ansari@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{Processes any link and gets meta info for it}
+  s.description = %q{Fetches meta and embed information for any link.}
+
+  s.rubyforge_project = "links_processor"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "open-uri"
+  s.add_runtime_dependency "oembed"
+  s.add_runtime_dependency "image_size"
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: links_processor
+version: !ruby/object:Gem::Version
+  version: 0.5.1
+  prerelease: 
+platform: ruby
+authors:
+- Shadab Ahmed
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-03-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &15446840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *15446840
+- !ruby/object:Gem::Dependency
+  name: open-uri
+  requirement: &15441940 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *15441940
+- !ruby/object:Gem::Dependency
+  name: oembed
+  requirement: &15441400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *15441400
+- !ruby/object:Gem::Dependency
+  name: image_size
+  requirement: &15440980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *15440980
+description: Fetches meta and embed information for any link.
+email:
+- shadab.ansari@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Rakefile
+- lib/links_processor.rb
+- lib/links_processor/version.rb
+- links_processor.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: links_processor
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Processes any link and gets meta info for it
+test_files: []