article_crux 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4713c573646d3a81816234e8c17272a388dc5c1f
+  data.tar.gz: d982b33fc5efd04c7318f9307cfcdda0ea6b0927
+SHA512:
+  metadata.gz: a4e2b7ed4066f1339c256d54414ac17949ca0a9350c46c82dff244d2fe744778780321858dce11b83c6780ed5673d65f83e3528f40ffb03cc348dfdcf5b51823
+  data.tar.gz: cb82bf19afbca30dfd8b96b50c577e8f4e9998e54db04f55f8c4491da72106ed88adf6d9ef1fca92bff369fc6a7411ab451edac87ae86e4507fbdba7a99beb65

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/nbproject/private/
+/nbproject/*

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in article_crux.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 amitsaxena
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# ArticleCrux
+
+The gem scrapes HTML of a URL and returns the title and cover image which most likey represents the article. It also returns an array of tags. The gem can be useful in scenarios where you want to display a short summary of the article, before the end-user lands on the actual article. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'article_crux'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install article_crux
+
+## Usage
+<pre lang="ruby"><code>
+require 'article_crux'
+
+ArticleCrux.fetch("https://techcrunch.com/2017/04/18/facebook-announces-react-fiber-a-rewrite-of-its-react-framework/")
+=> {:image=>"https://tctechcrunch2011.files.wordpress.com/2017/04/image-uploaded-from-ios-1.jpg?w=764&h=400&crop=1", :title=>"Facebook announces React Fiber, a rewrite of its React framework", :tags=>["developers", "F82017", "Facebook", "Javascript", "react", "React Fiber"]}
+
+# In case you want to pass a custom user Agent (server can whitelist specific User agents, as well as you might me blocked, and at times a server might return a different for different user agent)
+user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:10.0) Gecko/20100101 Firefox/10.0"
+ArticleCrux.fetch("https://techcrunch.com/2017/04/18/facebook-announces-react-fiber-a-rewrite-of-its-react-framework/", user_agent)
+=> {:image=>"https://tctechcrunch2011.files.wordpress.com/2017/04/image-uploaded-from-ios-1.jpg?w=764&h=400&crop=1", :title=>"Facebook announces React Fiber, a rewrite of its React framework", :tags=>["developers", "F82017", "Facebook", "Javascript", "react", "React Fiber"]}
+</code></pre>
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/amitsaxena/article_crux.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/article_crux.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'article_crux/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "article_crux"
+  spec.version       = ArticleCrux::VERSION
+  spec.authors       = ["amitsaxena"]
+  spec.email         = ["amit83.saxena@gmail.com"]
+
+  spec.summary       = %q{Returns crux of an article, which includes title and the most likely image that represents the article}
+  spec.description   = %q{The gem scrapes HTML of a URL to return the title and cover image which most likey represents the article. It also returns an array of tags.}
+  spec.homepage      = "https://github.com/amitsaxena/article_crux"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against " \
+      "public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+
+  spec.add_dependency "fastimage", "~> 2.0"
+  spec.add_dependency "nokogiri", "~> 1.6"
+  spec.add_dependency "httparty", "~> 0.12"
+  spec.add_dependency 'addressable', "~> 2.3"
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "article_crux"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/article_crux.rb

@@ -0,0 +1,117 @@
+require "article_crux/version"
+require 'httparty'
+require 'nokogiri'
+require 'fastimage'
+require 'addressable'
+require 'open-uri'
+
+module ArticleCrux
+  def self.fetch(url, user_agent="ArticleCrux(https://github.com/amitsaxena/article_crux)")
+    url = (url =~ /^(http|https):\/\/(.)*/i) ? url : "http://#{url}"
+    probe = HTTParty.head(url, headers: {"User-Agent" => user_agent})
+    if(probe.content_type && probe.content_type.split('/')[0] == "image")
+      return {:image => url, :title => nil, :tags => []}
+    end
+    
+    begin
+      res = HTTParty.get(url, headers: {"User-Agent" => user_agent})
+      raise "Unable to crawl URL" if res.code != 200
+      doc = Nokogiri::HTML(res)
+    rescue
+      doc = Nokogiri::HTML(open(url, "User-Agent" => user_agent))
+    end
+    
+    og_url = doc.search("//meta[@property='og:url' or @name='og:url']")
+    if (!og_url.empty? && !str_blank?(og_url[0]["content"]) && (og_url[0]["content"] =~ /^(http|https):\/\/(.)*/i) && (url != og_url[0]["content"]))
+      begin
+        res = HTTParty.get(og_url[0]["content"], headers: {"User-Agent" => user_agent})
+        raise "Unable to crawl URL" if res.code != 200
+        doc = Nokogiri::HTML(res)
+      rescue
+        # If the og:url is faulty or doesn't return a 200 response, use the original url for meta data scraping
+      end
+    end
+    
+    og_image = doc.search("//meta[@property='og:image' or @name='og:image']")
+    og_images = []
+    if !og_image.empty?
+      og_image.each do |ogi|
+        if !str_blank?(ogi["content"])
+          image = ogi["content"]
+          if (image =~ /^\/\/(.)*/)
+            uri = URI.parse(url)
+            image = "#{uri.scheme}:#{image}"
+          elsif (image =~ /^\/(.)*/)
+            uri = URI.parse(url)
+            image = File.join("#{uri.scheme}://#{uri.host}", image)
+          end
+          og_images << image
+        end
+      end
+    end
+
+    # Try to get the best image based on heuristics
+    image = get_best_image(og_images)
+
+    # If og:image is an invalid image (or extremely small), fall back to <img> tags
+    og_size = FastImage.size(Addressable::URI.escape(image)) if !str_blank?(image)
+    if (str_blank?(image) || og_size.nil? || (og_size[0] < 100 && og_size[1] < 100))
+      image = nil # reset image so that it doesn't show up in response
+      image_paths = []
+      page_images = doc.search("//img")
+      page_images.each do |page_image|
+        next if (str_blank?(page_image["src"]))
+        clip_image = page_image["src"]
+        if (clip_image && !(clip_image =~ /^(http|https):\/\/(.)*/i))
+          base = doc.search("//base")[0]
+          base_url = base["href"] if (!base.nil? && !str_blank?(base["href"]))
+          uri = URI.parse(url)
+          if (clip_image =~ /^\/(.)*/)
+            base_url = "#{uri.scheme}://#{uri.host}" if str_blank?(base_url)
+          else
+            base_url = "#{uri.scheme}://#{uri.host}#{uri.path[%r{^(.*[\/])}]}" if str_blank?(base_url)
+          end
+          clip_image = File.join(base_url, clip_image)
+        end
+        image_paths << clip_image
+      end
+    end
+
+    og_title = doc.search("//meta[@property='og:title' or @name='og:title']")
+    if (!og_title.empty? && !str_blank?(og_title[0]["content"]))
+      clip_title = og_title[0]["content"]
+    else
+      page_title = doc.search("//title")[0]
+      clip_title = page_title.text if !page_title.nil?
+    end
+
+    tags = []
+    possible_tags = doc.xpath('//meta[contains(@name, "tag") or contains(@name, "keyword") or contains(@property, "tag") or contains(@property, "keyword")]')
+    possible_tags.each{|e| tags << e["content"].split(',') if !str_blank?(e["content"])}
+    tags = tags.flatten.map(&:strip).uniq
+
+    res = {:image => image, :title => clip_title, :tags => tags}
+  end
+  
+  def self.get_best_image(images)
+    return nil if images.empty?
+    return images[0] if (images.size == 1)
+    # reject logo or similar images
+    refined_images = images.reject{|i| i =~ /logo|fallback/i}
+    return refined_images[0] if (refined_images.size == 1)
+    refined_images = images if refined_images.empty?
+    dimensions = []
+    refined_images.each do |i|
+      type = FastImage.type(Addressable::URI.escape(i))
+      size = FastImage.size(Addressable::URI.escape(i))
+      return i if((type == :gif) && (size && size[0] > 299 && size[1] > 199))
+      dimensions << {:x => size[0], :y => size[1], :image => i} if !size.nil?
+    end
+    image = dimensions.empty? ? nil : dimensions.max_by{|d| d[:x]}[:image]
+    return image
+  end
+  
+  def self.str_blank?(str)
+    str.nil? || str.empty?
+  end
+end

data/lib/article_crux/version.rb

@@ -0,0 +1,3 @@
+module ArticleCrux
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,156 @@
+--- !ruby/object:Gem::Specification
+name: article_crux
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- amitsaxena
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-04-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: fastimage
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.12'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+description: The gem scrapes HTML of a URL to return the title and cover image which
+  most likey represents the article. It also returns an array of tags.
+email:
+- amit83.saxena@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- article_crux.gemspec
+- bin/console
+- bin/setup
+- lib/article_crux.rb
+- lib/article_crux/version.rb
+homepage: https://github.com/amitsaxena/article_crux
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Returns crux of an article, which includes title and the most likely image
+  that represents the article
+test_files: []