raev 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3c9accf41425df615fb4f1b864fc8075fe0940b
-  data.tar.gz: d8c643c647b0ef2d46a02c23003a9d266ed1818e
+  metadata.gz: 0e99b301f4885c807426b83c31d4d9c5404c763e
+  data.tar.gz: cd0e1a7a522efdda1869c704625aa2921227696e
 SHA512:
-  metadata.gz: a107b4380e62a65f2ce83c1477a16f8f083d2bc32e4665fe63e74838f797af94c8e52292601aa7414e88da686ca6e78238c65e228d123ef1e6b38890c4c93eac
-  data.tar.gz: e2d2964362e7d42de36f44f0feb4b8265d418faa0245eeb2e7d47fb80a433f45993cd176b54ba52b1e0a2f4d7f9cc2c14d54fe88cfffe7a35d8b264afc6a1446
+  metadata.gz: e33460b3be3e68a48e6cec8d24e69f70595490959addc494e657ef12fb10f8085bd5dec51bc9c74d257e0ca12c5645bd6beca15c58dcb50f8b752dd81ae3eb27
+  data.tar.gz: bd0ec76ed70319849b758d4250a5dc3ad65f453e3138e0ffeedb074c8f704b756ff12be57c5670048dbb8f31ea59c62db59014699cc358ee6b8d2803aa7a0a5e

data/Gemfile

@@ -2,7 +2,6 @@ source "http://rubygems.org"
 
 gem "json", '>= 2.1.0'
 gem "nokogiri", ">= 1.8.0"
-gem "redirect_follower", ">= 0.1.1"
 gem "sanitize", ">= 2.1.0"
 gem "chronic", ">=0.10.2"
 

data/Gemfile.lock

@@ -60,7 +60,6 @@ GEM
     rack (2.0.3)
     rake (12.1.0)
     rdoc (5.1.0)
-    redirect_follower (0.1.1)
     sanitize (4.5.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.4.4)
@@ -87,7 +86,6 @@ DEPENDENCIES
   jeweler (= 2.3.7)
   json (>= 2.1.0)
   nokogiri (>= 1.8.0)
-  redirect_follower (>= 0.1.1)
   sanitize (>= 2.1.0)
   shoulda
   test-unit (= 3.2.4)

data/README.md

@@ -22,31 +22,24 @@ Usage
 Get the domain name from an url without the `www.` subdomain.
 
 ```ruby
-Raev.url("http://indiegames.com/2011/05/c418_minecraft_volume_alpha.html").base
+Raev::Url.base("http://indiegames.com/2011/05/c418_minecraft_volume_alpha.html")
 # => "indiegames.com"
 ```
 
 Remove UTM analytics parameters from an url.
 
 ```ruby
-Raev.url("http://ipodtouchlab.com/2011/01/iphone-ipad-app-sale-20110117.html?utm_campaign=touch_lab_bot&utm_medium=twitter&utm_source=am6_feedtweet").clean
-# =>  "http://ipodtouchlab.com/2011/01/iphone-ipad-app-sale-20110117.html"
+Raev::Url.remove_utm("http://www.ign.com/articles/2011/06/24/new-controllers-for-the-disabled-debuts-and-promises-change&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+ignfeeds%2Fgames+%28IGN+Videogames%29")
+# =>  "http://www.ign.com/articles/2011/06/24/new-controllers-for-the-disabled-debuts-and-promises-change"
 ```
 
 Resolve a shortened or proxied url.
 
 ```ruby
-Raev.url("http://sbn.to/WRgXfl").resolved
+Raev.url("http://sbn.to/WRgXfl").url
 # => "http://www.polygon.com/features/2013/3/25/4128022/gdc-gathering-of-game-makers"
 ```
 
-Resolve a shortend or proxied url and remove UTM analytics parameters.
-
-```ruby
-url = Raev.url("http://feedproxy.google.com/~r/fingergaming/~3/nBkNwBLq-U8/").resolved_and_clean 
-# => "http://www.gamasutra.com/topic/smartphone-tablet/fg/2011/01/21/zynga-acquires-drop7-developer-areacode/"   
-```
-
 Fetch Twitter handle from url.
 
 ```ruby

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.3.0

data/lib/raev/url.rb

@@ -1,6 +1,7 @@
 require "chronic"
 require "json"
 require "sanitize"
+require 'net/http'
 
 module Raev
   
@@ -33,48 +34,33 @@ module Raev
     REGEX_PAGE_TITLE = / +/
     
     attr_reader :url
+    attr_reader :body
     attr_reader :doc
   
     def initialize(url)
-      @url = url
+      fetch(url)
+      @url = Url.remove_utm(@url)
       @doc = nil
       @linked_data = nil
     end
 
-    def base      
-      base_url = @url.split('/'.freeze)[2]  
+    def self.base(url)
+      base_url = url.split('/'.freeze)[2]  
       base_url.gsub!('www.'.freeze, ''.freeze) unless base_url.nil?
       base_url
     end
     
-    def clean
-      unless @url.nil?
-        utm_index = @url.index(REGEX_UTM)
+    def self.remove_utm(url)
+      unless url.nil?
+        utm_index = url.index(REGEX_UTM)
         unless(utm_index.nil?)
-          return url.slice(0, utm_index)
+          url = url.slice(0, utm_index)
         end
       end
       
-      @url
+      url
     end
     
-    def resolved
-      unless @url.nil?
-        begin
-          return RedirectFollower(@url, 5)
-        rescue => ex
-          puts "Could not resolve #{@url}. #{ex.class}: #{ex.message}"
-        end
-      end
-
-      @url
-    end
-    
-    def resolved_and_clean
-      resolved_url = Url.new(self.resolved)
-      resolved_url.clean      
-    end
-
     def without_http
       @url.sub("http://".freeze, "".freeze)
     end
@@ -232,7 +218,7 @@ module Raev
 
     def document
       if @doc.nil?
-        @doc = Nokogiri::HTML(open(@url))
+        @doc = Nokogiri::HTML(@body)
       end
 
       @doc
@@ -252,5 +238,20 @@ module Raev
       @linked_data
     end
     
+    def fetch(uri_str, limit = 10)
+      raise ArgumentError, 'too many HTTP redirects' if limit == 0
+
+      response = Net::HTTP.get_response(URI(uri_str))
+
+      case response
+      when Net::HTTPSuccess then
+        @url = uri_str
+        @body = response.body
+      when Net::HTTPRedirection then
+        fetch(response['location'], limit - 1)
+      else
+        # TODO handle Not Found
+      end
+    end
   end
 end

data/lib/raev.rb

@@ -1,7 +1,4 @@
 require 'nokogiri'
-require 'redirect_follower'
-require 'open-uri'
-
 require 'raev/article'
 require 'raev/author'
 require 'raev/parser'

data/raev.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: raev 0.2.4 ruby lib
+# stub: raev 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "raev".freeze
-  s.version = "0.2.4"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
   s.authors = ["Andreas Zecher".freeze]
-  s.date = "2017-09-18"
+  s.date = "2017-09-19"
   s.description = "Fetch, parse and normalize meta data from websites.".freeze
   s.email = "andreas@madebypixelate.com".freeze
   s.extra_rdoc_files = [
@@ -48,7 +48,6 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<json>.freeze, [">= 2.1.0"])
       s.add_runtime_dependency(%q<nokogiri>.freeze, [">= 1.8.0"])
-      s.add_runtime_dependency(%q<redirect_follower>.freeze, [">= 0.1.1"])
       s.add_runtime_dependency(%q<sanitize>.freeze, [">= 2.1.0"])
       s.add_runtime_dependency(%q<chronic>.freeze, [">= 0.10.2"])
       s.add_development_dependency(%q<shoulda>.freeze, [">= 0"])
@@ -58,7 +57,6 @@ Gem::Specification.new do |s|
     else
       s.add_dependency(%q<json>.freeze, [">= 2.1.0"])
       s.add_dependency(%q<nokogiri>.freeze, [">= 1.8.0"])
-      s.add_dependency(%q<redirect_follower>.freeze, [">= 0.1.1"])
       s.add_dependency(%q<sanitize>.freeze, [">= 2.1.0"])
       s.add_dependency(%q<chronic>.freeze, [">= 0.10.2"])
       s.add_dependency(%q<shoulda>.freeze, [">= 0"])
@@ -69,7 +67,6 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<json>.freeze, [">= 2.1.0"])
     s.add_dependency(%q<nokogiri>.freeze, [">= 1.8.0"])
-    s.add_dependency(%q<redirect_follower>.freeze, [">= 0.1.1"])
     s.add_dependency(%q<sanitize>.freeze, [">= 2.1.0"])
     s.add_dependency(%q<chronic>.freeze, [">= 0.10.2"])
     s.add_dependency(%q<shoulda>.freeze, [">= 0"])

data/test/test_url.rb

@@ -5,29 +5,20 @@ require 'raev'
 
 class UrlTest < Test::Unit::TestCase
   should "parse base url" do
-    url = Raev.url("http://indiegames.com/2011/05/c418_minecraft_volume_alpha.html")
-    assert_equal url.base, "indiegames.com"
+    assert_equal Raev::Url.base("http://indiegames.com/2011/05/c418_minecraft_volume_alpha.html"), "indiegames.com"
   end
   
   should "clean url" do
-    url = Raev.url("http://ipodtouchlab.com/2011/01/iphone-ipad-app-sale-20110117.html?utm_campaign=touch_lab_bot&utm_medium=twitter&utm_source=am6_feedtweet")
-    assert_equal "http://ipodtouchlab.com/2011/01/iphone-ipad-app-sale-20110117.html", url.clean
-    
-    url = Raev.url("http://games.ign.com/articles/117/1178937p1.html?RSSwhen2011-06-24_082700&RSSid=1178937&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+ignfeeds%2Fgames+%28IGN+Videogames%29")
-    assert_equal "http://games.ign.com/articles/117/1178937p1.html?RSSwhen2011-06-24_082700&RSSid=1178937", url.clean 
-    
-    url = Raev.url("http://boingboing.net/2011/08/09/ea-tried-to-buy-minecraft-studio.html")
-    assert_equal "http://boingboing.net/2011/08/09/ea-tried-to-buy-minecraft-studio.html", url.clean
+    cleaned_url = Raev::Url.remove_utm("http://www.ign.com/articles/2011/06/24/new-controllers-for-the-disabled-debuts-and-promises-change?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+ignfeeds%2Fgames+%28IGN+Videogames%29")
+    assert_equal "http://www.ign.com/articles/2011/06/24/new-controllers-for-the-disabled-debuts-and-promises-change", cleaned_url
   end  
-  
-  should "resolve url" do
-    url = Raev.url("http://feedproxy.google.com/~r/fingergaming/~3/nBkNwBLq-U8/")
-    assert_equal "https://www.gamasutra.com/topic/smartphone-tablet/fg/2011/01/21/zynga-acquires-drop7-developer-areacode/?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+fingergaming+%28FingerGaming%29", url.resolved
-  end
-  
-  should "resolve and clean url" do 
+    
+  should "resolve url" do 
     url = Raev.url("http://feedproxy.google.com/~r/fingergaming/~3/nBkNwBLq-U8/")
-    assert_equal "https://www.gamasutra.com/topic/smartphone-tablet/fg/2011/01/21/zynga-acquires-drop7-developer-areacode/", url.resolved_and_clean    
+    assert_equal "https://www.gamasutra.com/topic/smartphone-tablet/fg/2011/01/21/zynga-acquires-drop7-developer-areacode/", url.url    
+
+    url = Raev.url("http://boingboing.net/2011/08/09/ea-tried-to-buy-minecraft-studio.html")
+    assert_equal "https://boingboing.net/2011/08/09/ea-tried-to-buy-minecraft-studio.html", url.url
   end
   
   should "get twitter handle" do
@@ -36,8 +27,8 @@ class UrlTest < Test::Unit::TestCase
   end
   
   should "get rss feed" do
-    url = Raev.url("http://www.polygon.com")
-    assert_equal "http://www.polygon.com/rss/index.xml", url.feed
+    url = Raev.url("https://www.polygon.com")
+    assert_equal "https://www.polygon.com/rss/index.xml", url.feed
     
     url = Raev.url("http://arstechnica.com")
     assert_equal "http://feeds.arstechnica.com/arstechnica/index/", url.feed

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: raev
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Andreas Zecher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-18 00:00:00.000000000 Z
+date: 2017-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -38,20 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.8.0
-- !ruby/object:Gem::Dependency
-  name: redirect_follower
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.1
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.1.1
 - !ruby/object:Gem::Dependency
   name: sanitize
   requirement: !ruby/object:Gem::Requirement