brilliant_web_scraper 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: efbe9d1a0688fd10e200d972b56c3e2ec86203f1
+  data.tar.gz: 20cce1c52197f11dcea73813831bb4172829ddaa
+SHA512:
+  metadata.gz: 638c34f7efbc963613f4bb841abbf183bf134ee3197bebc99f9403ba7864befd44243f53092a9aa3ba7ea58314475b61d6671816e8e3f8ef4deb7f49b6f0ef52
+  data.tar.gz: f91110f69e8228de408aa0c35050fe6137fac22bdb93ff86be3c70d380e1cf57534f50f78e5d585cb63e421ff0fc51aa04089d38e8a86ab3a6ca305659dc909a

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in web_scraping.gemspec
+gemspec

data/README.md

@@ -0,0 +1,31 @@
+# WebScraper [![Build Status](https://api.travis-ci.com/bkotu6717/brilliant_web_scraper.svg)](https://travis-ci.com/bkotu6717/brilliant_web_scraper)
+
+A decent web scraping gem. Scrapes website description, social profiles, contact details, youtube channels.
+
+
+It accepts a URL or Domain as input and gets it's title, descrptios, social profiles, YouTube channels and it's current URL if got redirected.
+
+
+## See it in action!
+
+You can try WebScraper live at this little demo: [https://brilliantweb-scraper-demo.herokuapp.com](https://brilliant-web-scraper-demo.herokuapp.com)
+
+## Installation
+
+
+If you're using it on a Rails application, just add it to your Gemfile and run `bundle install`
+
+```ruby
+gem 'brilliant_web_scraper'
+```
+
+## Usage
+
+Initialize a BrilliantWebScraper instance for an URL, like this:
+
+```ruby
+require 'brilliant_web_scraper'
+results = BrilliantWebScraper.new('http://pwc.com')
+```
+
+If you don't include the scheme on the URL, it is fine:

data/brilliant_web_scraper.gemspec

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require File.expand_path('./lib/version')
+
+Gem::Specification.new do |s|
+  s.name =  'brilliant_web_scraper'
+  s.version = WebScraper::VERSION
+  s.licenses = ['Nonstandard']
+  s.summary = 'A decent web scraping ruby library!'
+  s.description = 'Scrapes data such as description, social profiles, contact details'
+  s.authors = ['Kotu Bhaskara Rao']
+  s.email = 'bkotu6717@gmail.com'
+  s.require_paths = ['lib']
+  s.homepage = 'https://github.com/bkotu6717/brilliant_web_scraper'
+  s.files = Dir['**/*'].keep_if { |file|
+    file != "brilliant_web_scraper-#{WebScraper::VERSION}.gem" && File.file?(file)
+  }
+  s.required_ruby_version = '>= 2.3.0'
+
+  s.add_dependency 'nesty', '~> 1.0', '>= 1.0.1'
+  s.add_dependency 'rest-client', '~> 2.0', '>= 2.0.2'
+
+  s.add_development_dependency 'nesty', '~> 1.0', '>= 1.0.1'
+  s.add_development_dependency 'pry', '~> 0.12.2'
+  s.add_development_dependency 'rest-client', '~> 2.0', '>= 2.0.2'
+  s.add_development_dependency 'rspec', '~> 3.5'
+  s.add_development_dependency 'rubocop', '~> 0.73.0'
+  s.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.1'
+  s.add_development_dependency 'webmock', '~> 2.1'
+end

data/lib/brilliant_web_scraper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'rest-client'
+require 'cgi'
+require 'benchmark'
+
+current_directory = File.dirname(__FILE__) + '/scraper'
+require File.expand_path(File.join(current_directory, 'errors'))
+require File.expand_path(File.join(current_directory, 'scrape_exceptions'))
+require File.expand_path(File.join(current_directory, 'scrape_helper'))
+require File.expand_path(File.join(current_directory, 'scrape_request'))
+
+current_directory = File.dirname(__FILE__) + '/parsers'
+require File.expand_path(File.join(current_directory, 'unescape_html_helper'))
+require File.expand_path(File.join(current_directory, 'description_helper'))
+require File.expand_path(File.join(current_directory, 'title'))
+require File.expand_path(File.join(current_directory, 'meta_description'))
+require File.expand_path(File.join(current_directory, 'org_description'))
+require File.expand_path(File.join(current_directory, 'twitter_description'))
+require File.expand_path(File.join(current_directory, 'twitter_profile'))
+require File.expand_path(File.join(current_directory, 'linkedin_profile'))
+require File.expand_path(File.join(current_directory, 'facebook_profile'))
+require File.expand_path(File.join(current_directory, 'youtube_channel'))
+require File.expand_path(File.join(current_directory, 'instagram_profile'))
+require File.expand_path(File.join(current_directory, 'vimeo_profile'))
+require File.expand_path(File.join(current_directory, 'pinterest_profile'))
+require File.expand_path(File.join(current_directory, 'emails'))
+require File.expand_path(File.join(current_directory, 'phone_numbers'))
+require File.expand_path(File.join(current_directory, 'redirected_to'))
+
+# Main scraping class
+class BrilliantWebScraper
+  extend ScrapeHelper
+  extend ScrapeRequest
+  extend Title
+  extend MetaDescription
+  extend OrgDescription
+  extend TwitterDescription
+  extend TwitterProfile
+  extend LinkedinProfile
+  extend FacebookProfile
+  extend YoutubeChannel
+  extend InstagramProfile
+  extend VimeoProfile
+  extend PinterestProfile
+  extend Emails
+  extend PhoneNumbers
+  extend RedirectedTo
+
+  class << self
+    def new(url, connection_timeout = 10, read_timeout = 10)
+      perform_scrape(url, connection_timeout, read_timeout)
+    end
+  end
+end

data/lib/parsers/description_helper.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# @Parses meta, twitter, org description tags
+module DescriptionHelper
+  include UnescapeHtmlHelper
+
+  private
+
+  def scrape_description(response, regexes)
+    return if response.to_s.empty? || regexes.empty?
+
+    description = nil
+    regexes.each do |regex|
+      description = response.scan(regex).flatten.compact
+      description = parse_description(description)
+      break unless description.nil?
+    end
+    unescape_html(description)
+  end
+
+  def parse_description(descriptions)
+    return if descriptions.nil? || descriptions.empty?
+
+    descriptions = descriptions.reject { |x| x.nil? || x.empty? }
+    descriptions = descriptions.map { |x| unescape_html(x) }
+    descriptions.find { |x| (x !~ /^\s*[|-]?\s*$/) }
+  end
+end

data/lib/parsers/emails.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+current_directory = File.dirname(__FILE__)
+require File.expand_path(File.join(current_directory, 'unescape_html_helper'))
+
+# Parses emails from html string
+module Emails
+  include UnescapeHtmlHelper
+  def grep_emails(response)
+    return if response.nil? || response.empty?
+
+    first_regex = /(?im)mailto:\s*([^\?"',\\<>\s]+)/
+    second_regex = %r{(?im)["'\s><\/]*([\w._%-]+@(?!(?:example|e?mail|domain|company|your(?:domain|company|email)|address|emailad?dress|yyy|test)\.)[\w._%-]+\.(?!png|jpe?g|tif|svg)[A-Z]{2,3})["'\s><]}
+    first_set = response.scan(first_regex).flatten.compact
+    first_set = get_processed_emails(first_set)
+    second_set = response.scan(second_regex).flatten.compact
+    second_set = get_processed_emails(second_set)
+    (first_set | second_set).compact.map(&:downcase).uniq
+  end
+
+  def get_processed_emails(email_set)
+    return [] if email_set.nil? || email_set.empty?
+
+    unescaped_emails = email_set.map { |email| unescape_html(email) }
+    return [] if unescaped_emails.empty?
+
+    email_match_regex = /[\w._%-]+@(?!(?:example|e?mail|domain|company|your(?:domain|company|email)|address|emailad?dress|yyy|test)\.)[\w._%-]+\.(?!png|jpe?g|tif|svg)[A-Z]{2,3}/im
+    unescaped_emails.select { |data| data =~ email_match_regex }
+  end
+end

data/lib/parsers/facebook_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep facebook profiles
+module FacebookProfile
+  def grep_facebook_profile(response)
+    return if response.nil? || response.empty?
+
+    facebook_url_regex = /(https?:\/\/(?:www\.)?(?:facebook|fb)\.com\/(?!tr\?|(?:[\/\w\d]*(?:photo|sharer?|like(?:box)?|offsite_event|plugins|permalink|home|search))\.php|\d+\/fbml|(?:dialog|hashtag|plugins|sharer|login|recover|security|help|v\d+\.\d+)\/|(?:privacy|#|your-profile|yourfacebookpage)\/?|home\?)[^"'<>\&\s]+)/im
+    response.scan(facebook_url_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/instagram_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep instgram profiles
+module InstagramProfile
+  def grep_instagram_profile(response)
+    return if response.nil? || response.empty?
+
+    instagram_regex = %r{(?im)(https?:\/\/(?:www\.)?+instagram\.com\/(?!#|%|"|'|(?:explore|p)\/).+?[^"'<>\s?&\/]+)}
+    response.scan(instagram_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/linkedin_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep linkedin profile
+module LinkedinProfile
+  def grep_linkedin_profile(response)
+    return if response.nil? || response.empty?
+
+    linkedin_profile_regex = %r{(?im)(https:\/\/www\.linkedin\.com\/company\/[^"'\?<>\s\/]+)}
+    response.scan(linkedin_profile_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/meta_description.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Grep description in meta tag with attribute name='description'
+module MetaDescription
+  include DescriptionHelper
+  def grep_meta_description(response)
+    return if response.nil? || response.empty?
+
+    first_regex = %r{(?im)(?im)<meta\s+[\w\s"'=-]*(?:name|itemprop)\s*=\s*(?:'|")?\s*description\s*(?:'|")?[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*[\/>]}
+    second_regex = %r{(?im)<meta\s+[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*(?:name|itemprop)\s*=\s*(?:'|")?\s*description\s*(?:'|")?[\w\s"'=-]*[\/>]}
+    scrape_description(response, [first_regex, second_regex])
+  end
+end

data/lib/parsers/org_description.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Greps description from meta tag with attrbute org:description
+module OrgDescription
+  include DescriptionHelper
+  def grep_org_description(response)
+    return if response.nil? || response.empty?
+
+    first_regex = %r{(?im)<meta\s+[\w\s"'=-]*(?:property|itemprop)\s*=\s*(?:'|")?\s*og:description\s*(?:'|")?[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*[\/>]}
+    second_regex = %r{(?im)<meta\s+[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*(?:property|itemprop)\s*=\s*(?:'|")?\s*og:description\s*(?:'|")?[\w\s"'=-]*[\/>]}
+    scrape_description(response, [first_regex, second_regex])
+  end
+end

data/lib/parsers/phone_numbers.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# Grep phonenumbers from 'href=tel:' attributes
+module PhoneNumbers
+  include UnescapeHtmlHelper
+  def grep_phone_numbers(response)
+    return if response.nil? || response.empty?
+
+    phone_number_regex = %r{(?im)href\s*=\s*(?:"|')?\s*tel:\s*(?:https?:)?\/*(?!#(?:"|'))([^"'\/<>\{\[]+)}
+    phone_numbers = response.scan(phone_number_regex).flatten.uniq
+    get_processed_phone_numbers(phone_numbers)
+  end
+
+  private
+
+  def get_processed_phone_numbers(phone_numbers)
+    return [] if phone_numbers.nil? || phone_numbers.empty?
+
+    unescaped_contacts = phone_numbers.map { |phone_number| unescape_html(phone_number) }
+    good_phone_numbers = []
+    unescaped_contacts.each do |x|
+      next if x !~ /\d+/
+
+      if x =~ /\w+=/
+        good_phone_numbers << x.gsub(/\w+=.*/, '')
+        next
+
+      else
+        good_phone_numbers << x
+      end
+    end
+    good_phone_numbers.uniq
+  end
+end

data/lib/parsers/pinterest_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep pinterest profile
+module PinterestProfile
+  def grep_pinterest_profile(response)
+    return if response.nil? || response.empty?
+
+    pinterest_regex = %r{(?im)(https?:\/\/[\w\.]*pinterest\.com\/(?!"|'|\?|#|cookies(?:"|'')|(?:pin|v3|js|feed)\/)[^"'<>?&\s\/]+)}
+    response.scan(pinterest_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/redirected_to.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Fetch latest url of the given website
+module RedirectedTo
+
+  def grep_redirected_to_url(response)
+    return if response.nil? || response.empty?
+
+    patterns = [
+      %r{(?im)<link\s+[\s\w="'-]*rel\s*=\s*(?:"|')canonical(?:"|')[\s\w='"-]*?\s+href\s*=\s*(?:"|')([^"']*)(?:"|')[\s\w='"-]*?(?:>|\/>)},
+      %r{(?im)<link\s+[\s\w='"-]*href\s*=\s*(?:"|')([^'"]*)(?:"|')[\s\w='"-]*?rel\s*=\s*(?:"|')\s*canonical\s*(?:"|')[\s\w='"-]*(?:>|\/>)},
+      %r{(?im)<meta\s+[\s\w="'-]*property=\s*(?:'|")\s*og:url\s*(?:'|")[\s\w="'-]*content=\s*(?:'|")([^'"]*)(?:'|")[\s\w="'-]*(?:>|\/>)},
+      %r{(?im)<meta\s+[\s\w"'=-]*content\s*=\s*(?:'|")([^'"]*)(?:'|")[\s\w"'=-]*property\s*=\s*(?:'|")\s*og:url\s*(?:'|")[\s\w"'=-]*(?:>|\/>)}
+    ]
+    url = nil
+    patterns.each do |pattern|
+      web_urls = response.scan(pattern).flatten
+      url = parser(web_urls)
+      break unless url.nil?
+    end
+    url
+  end
+
+  private
+
+  def parser(urls)
+    urls.find { |x| x =~ %r{(?im)^\s*(?:https*)?:?(?:\/\/)?\w+[.&%-]} }
+  end
+end

data/lib/parsers/title.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Grep title form very first title tag
+module Title
+	include UnescapeHtmlHelper
+	def grep_title(response)
+		return if !response.is_a?(String) || response.empty?
+
+		title_regex =  /<\s*title.*?>(.*?)<?\s*\/?title\s*?>/im
+    title = response.match(title_regex).captures[0].strip rescue nil
+		unescape_html(title) unless title.nil? || title.empty?
+	end
+end

data/lib/parsers/twitter_description.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# Grep twitter description from attribute `twitter:description`
+module TwitterDescription
+  include DescriptionHelper
+  def grep_twitter_description(response)
+    return if response.nil? || response.empty?
+
+    first_regex = %r{(?im)<meta\s+[\w\s"'=-]*(?:name|itemprop)\s*=\s*(?:'|")?\s*twitter:description\s*(?:'|")?[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*[\/>]}
+    second_regex = %r{(?im)<meta\s+[\w\s"'=-]*content\s*=\s*(?:(?:"([^"]*)")|(?:'([^']*)'))[\w\s"'=-]*(?:name|itemprop)\s*=\s*(?:'|")?\s*twitter:description\s*(?:'|")?[\w\s"'=-]*[\/>]}
+    scrape_description(response, [first_regex, second_regex])
+  end
+end

data/lib/parsers/twitter_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep twitter profile
+module TwitterProfile
+  def grep_twitter_profile(response)
+    return if response.nil? || response.empty?
+
+    twitter_regex = %r{(?im)(https?:\/\/(?:www\.)?twitter\.com\/(?!(?:share|download|search|home|login|privacy)(?:\?|\/|\b)|(?:hashtag|i|javascripts|statuses|#!|intent)\/|(?:#|'|%))[^"'&\?<>\s\\]+)}
+    response.scan(twitter_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/unescape_html_helper.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Decode HTML & URL encodings
+module UnescapeHtmlHelper
+  private
+
+  def unescape_html(text)
+    return if text.nil? && !text.is_a?(String) || text.empty?
+
+    unescaped_html_text = CGI.unescapeHTML(text)
+    if unescaped_html_text =~ /%[a-z0-9]{2}/i
+      plus_sign_preserved_text = unescaped_html_text.gsub(/\+/, '%2B')
+      unescaped_html_text = CGI.unescape(plus_sign_preserved_text)
+    end
+    unescaped_html_text.strip
+  end
+end

data/lib/parsers/vimeo_profile.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Grep Vimeo social profile
+module VimeoProfile
+  def grep_vimeo_profile(response)
+    return if response.nil? || response.empty?
+
+    vimeo_regex = %r{(?im)(https?:\/\/(?:www\.)?vimeo\.com\/(?!upgrade|features|enterprise|upload|api)\/?[^"'\&\?<>\s]+)}
+    response.scan(vimeo_regex).flatten.compact.uniq
+  end
+end

data/lib/parsers/youtube_channel.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Grep youtube channels
+module YoutubeChannel
+  def grep_youtube_channel(response)
+    return if response !~ %r{(?im)https?:\/\/(?:www\.)?youtube\.com\/}
+
+    first_regex = %r{(?im)(https?:\/\/(?:www\.)?youtube\.com\/(?!\?gl=\w{2}|(?:embed|feeds)\/|(?:player_api|iframe_api)(?:"|'|\/|\?)|watch\?|user\/#)[^"'\&<>\s]+)}
+    second_regex = %r{(?im)(https?:\/\/(?:www\.)?youtube\.com\/watch?\S*v=[^<>&'"]+)}
+    third_regex = %r{(?im)(https?:\/\/(?:www\.)?youtube\.com\/embed\/(?!id|{|}|\[|\]|\$|\?|\\|%|\+)[^"'\?<>\s]+)}
+    youtube_channels = scrape_profile(response, [first_regex, second_regex, third_regex])
+    youtube_channels.compact.uniq
+  end
+
+  private
+
+  def scrape_profile(response, regexes)
+    return if response.to_s.empty? || regexes.empty?
+
+    profiles = []
+    regexes.each do |regex|
+      profiles = response.scan(regex).flatten.compact
+      break unless profiles.empty?
+    end
+    return [] if profiles.none?
+
+    profiles
+  end
+end