raev 0.1.12 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55c78f5f5edfe18398bc184a0a8268978182d09f
-  data.tar.gz: 550566eba663b2f9cc5930f98be4f95a9c023ddd
+  metadata.gz: 64988202ffa1289c5d9e0f6fe59767e1bf6c2b2d
+  data.tar.gz: 1fc08d2929d84fbb849a81c6bd4a843eb952b644
 SHA512:
-  metadata.gz: c83322a8f28c89ad3e28fa7739b244a17c711c68cca5e5d2060f7403ad44b7323eece461c69659f345916d5908c401a7e6a898ce73f1dd9f855a4344334bfa3f
-  data.tar.gz: 529f05c783b388f391a22864efb10968d879163bbb8b41bf02f2d61decd83eb07780eb6a4aaccf7298b1729006c1ac3c69b9f50a024436b36d98ce95238a9fab
+  metadata.gz: 42cf3c6f84cc0061b7b5b51e43796546c9e03d1a5064d20b187783f065a354a63b90cbe8ae91254436a915d75c0459dae6eabfb6a9ffbb95e6197add643245fa
+  data.tar.gz: f97e2730ca39ad035a0891701cdfc65cf7b1208cd313299a26146079ecaf38d5f4b9f69a36e23017be70f58153452cdbb4391b44c1b4dfcedfeb193ca15c0874

data/Gemfile

@@ -1,11 +1,14 @@
 source "http://rubygems.org"
 
+gem "json", '>= 1.8.1'
 gem "nokogiri", ">= 1.6.1"
 gem "redirect_follower", ">= 0.1.1"
+gem "sanitize", ">= 2.1.0"
+gem "chronic", ">=0.9.1"
 
 group :development do
   gem "shoulda", ">= 0"
-  gem "bundler", "~> 1.6.3"
+  gem "bundler", "~> 1.7.2"
   gem "jeweler", "2.0.1"
   gem "test-unit", "~> 2.5.4"
 end

data/Gemfile.lock

@@ -9,6 +9,8 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.3.6)
     builder (3.2.2)
+    chronic (0.10.2)
+    crass (1.0.2)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     faraday (0.9.0)
@@ -43,6 +45,8 @@ GEM
     multipart-post (2.0.0)
     nokogiri (1.6.4.1)
       mini_portile (~> 0.6.0)
+    nokogumbo (1.4.1)
+      nokogiri
     oauth2 (1.0.0)
       faraday (>= 0.8, < 0.10)
       jwt (~> 1.0)
@@ -54,6 +58,10 @@ GEM
     rdoc (4.1.2)
       json (~> 1.4)
     redirect_follower (0.1.1)
+    sanitize (4.0.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.4.4)
+      nokogumbo (= 1.4.1)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
       shoulda-matchers (>= 1.4.1, < 3.0)
@@ -69,9 +77,12 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  bundler (~> 1.6.3)
+  bundler (~> 1.7.2)
+  chronic (>= 0.9.1)
   jeweler (= 2.0.1)
+  json (>= 1.8.1)
   nokogiri (>= 1.6.1)
   redirect_follower (>= 0.1.1)
+  sanitize (>= 2.1.0)
   shoulda
   test-unit (~> 2.5.4)

data/Rakefile

@@ -21,10 +21,6 @@ Jeweler::Tasks.new do |gem|
   gem.description = "Fetch, parse and normalize meta data from websites."
   gem.email = "andreas@madebypixelate.com"
   gem.authors = ["Andreas Zecher"]
-  
-  # dependencies defined in Gemfile
-  gem.add_dependency 'nokogiri', '>= 1.4.4'
-  gem.add_dependency 'redirect_follower', '>= 0.1.1'  
 end
 Jeweler::RubygemsDotOrgTasks.new
 

data/VERSION

@@ -1 +1 @@
-0.1.12
+0.2.1

data/lib/raev/url.rb

@@ -1,13 +1,18 @@
+require "chronic"
+require "json"
+require "sanitize"
+
 module Raev
   
   class Url
-  
     attr_reader :url
     attr_reader :doc
+    attr_reader :linked_data
   
     def initialize(url)
-      @doc = nil
       @url = url
+      @doc = nil
+      @linked_data = nil
     end
 
     def base      
@@ -82,6 +87,10 @@ module Raev
     end
     
     def headline
+      if linked_data && linked_data["headline"]
+        return Sanitize.clean(linked_data["headline"])
+      end
+      
       page_title = nil
       
       node = document.css(".twitter-share-button")
@@ -114,15 +123,88 @@ module Raev
       
       page_title
     end
+        
+    def pubdate      
+      if linked_data && linked_data["datePublished"]
+        return Date.parse(linked_data["datePublished"])
+      end
+      
+      date_elements = @url.match(/[0-9]{4}\/[0-9]{1,2}\/[0-9]{1,2}/).to_s.split("/")
+      
+      if date_elements.size == 3
+        return Date.new(date_elements[0].to_i, date_elements[1].to_i, date_elements[2].to_i)      
+      else
+        node = document.search("meta[itemprop='datePublished'], meta[name='pub_date']").first
+        
+        if node
+          return Date.parse(node.attribute("content"))
+        else
+          node = document.search(".entryDate, .entrydate").first
+
+          if node
+            return Chronic.parse(node.content.gsub(/[^a-zA-Z0-9\s]/,"").strip)
+          end
+        end
+      end
+      
+      nil
+    end
     
+    def author
+      cssSelectors = [
+        '.author-info .name',
+        '.author-top a',
+        '.yt-user-name',
+        'a[rel~="author"]',
+        'a[itemprop~="author"]',
+        '.author h3 a',
+        '.author',
+        '.posted-by a',
+        '.entryAuthor a',
+        'a.names',
+        'a.byline-author',
+        '.byline a',
+        '.author.vcard a',
+        'p.info a',
+        '.author-name',
+        '.upcased',
+        'a[rel~="nofollow"]'
+      ]
+
+      node = document.search(cssSelectors.join(", ")).first
+      
+      if node
+        words = node.content.split.size
+      
+        if words <= 4
+          return Sanitize.clean(node.content).strip[0..255]
+        end
+      end
+      
+      ""
+    end
+        
     private
     
     def document
       if @doc.nil?
         @doc = Nokogiri::HTML(open(@url))
-      else
-        @doc
       end
+
+      @doc
+    end
+    
+    def linked_data
+      if @linked_data.nil?
+        node = document.css("script[type=\"application/ld+json\"]")
+      
+        if node.first
+          @linked_data = JSON.parse(node.first.content)
+        end
+      end
+      
+      @linked_data
     end
+    
   end
 end

data/raev.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: raev 0.2.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "raev"
-  s.version = "0.1.12"
+  s.version = "0.2.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Andreas Zecher"]
-  s.date = "2014-11-26"
+  s.date = "2015-08-20"
   s.description = "Fetch, parse and normalize meta data from websites."
   s.email = "andreas@madebypixelate.com"
   s.extra_rdoc_files = [
@@ -37,41 +39,43 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/pixelate/raev"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.3"
+  s.rubygems_version = "2.4.5"
   s.summary = "Fetch, parse and normalize meta data from websites."
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, [">= 1.8.1"])
       s.add_runtime_dependency(%q<nokogiri>, [">= 1.6.1"])
       s.add_runtime_dependency(%q<redirect_follower>, [">= 0.1.1"])
+      s.add_runtime_dependency(%q<sanitize>, [">= 2.1.0"])
+      s.add_runtime_dependency(%q<chronic>, [">= 0.9.1"])
       s.add_development_dependency(%q<shoulda>, [">= 0"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.6.3"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.7.2"])
       s.add_development_dependency(%q<jeweler>, ["= 2.0.1"])
       s.add_development_dependency(%q<test-unit>, ["~> 2.5.4"])
-      s.add_runtime_dependency(%q<nokogiri>, [">= 1.4.4"])
-      s.add_runtime_dependency(%q<redirect_follower>, [">= 0.1.1"])
     else
+      s.add_dependency(%q<json>, [">= 1.8.1"])
       s.add_dependency(%q<nokogiri>, [">= 1.6.1"])
       s.add_dependency(%q<redirect_follower>, [">= 0.1.1"])
+      s.add_dependency(%q<sanitize>, [">= 2.1.0"])
+      s.add_dependency(%q<chronic>, [">= 0.9.1"])
       s.add_dependency(%q<shoulda>, [">= 0"])
-      s.add_dependency(%q<bundler>, ["~> 1.6.3"])
+      s.add_dependency(%q<bundler>, ["~> 1.7.2"])
       s.add_dependency(%q<jeweler>, ["= 2.0.1"])
       s.add_dependency(%q<test-unit>, ["~> 2.5.4"])
-      s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
-      s.add_dependency(%q<redirect_follower>, [">= 0.1.1"])
     end
   else
+    s.add_dependency(%q<json>, [">= 1.8.1"])
     s.add_dependency(%q<nokogiri>, [">= 1.6.1"])
     s.add_dependency(%q<redirect_follower>, [">= 0.1.1"])
+    s.add_dependency(%q<sanitize>, [">= 2.1.0"])
+    s.add_dependency(%q<chronic>, [">= 0.9.1"])
     s.add_dependency(%q<shoulda>, [">= 0"])
-    s.add_dependency(%q<bundler>, ["~> 1.6.3"])
+    s.add_dependency(%q<bundler>, ["~> 1.7.2"])
     s.add_dependency(%q<jeweler>, ["= 2.0.1"])
     s.add_dependency(%q<test-unit>, ["~> 2.5.4"])
-    s.add_dependency(%q<nokogiri>, [">= 1.4.4"])
-    s.add_dependency(%q<redirect_follower>, [">= 0.1.1"])
   end
 end
 

data/test/test_url.rb

@@ -42,8 +42,8 @@ class UrlTest < Test::Unit::TestCase
     url = Raev.url("http://arstechnica.com")
     assert_equal "http://feeds.arstechnica.com/arstechnica/index/", url.feed
   
-    url = Raev.url("http://www.edge-online.com")
-    assert_equal "http://www.edge-online.com/feed/", url.feed
+    url = Raev.url("http://www.kotaku.com")
+    assert_equal "http://feeds.gawker.com/kotaku/full", url.feed
   end
   
   should "get headline from url" do
@@ -59,9 +59,6 @@ class UrlTest < Test::Unit::TestCase
     url = Raev.url("http://www.creativeapplications.net/games/below-new-from-the-creators-of-sword-sworcery/")
     assert_equal "Below – New from the creators of Sword & Sworcery", url.headline
 
-    url = Raev.url("http://www.edge-online.com/news/e3-2013-watch-the-first-hohokum-gameplay-footage-here/")
-    assert_equal "E3 2013: watch the first Hohokum gameplay footage here", url.headline
-
     url = Raev.url("http://www.giantbomb.com/videos/e3-2013-fez-ii-announcement-teaser/2300-7606/")
     assert_equal "E3 2013: Fez II Announcement Teaser", url.headline
 
@@ -74,4 +71,51 @@ class UrlTest < Test::Unit::TestCase
     url = Raev.url("http://www.rockpapershotgun.com/2013/06/05/i-spy-an-open-beta-for-spy-party/")
     assert_equal "I Spy An Open Beta For Spy Party", url.headline
   end
+  
+  should "get pubdate from url" do
+    url = Raev.url("http://www.polygon.com/2015/5/18/8620223/witcher-3-guide-witcher-2-witcher")
+    assert_equal_date Date.new(2015, 5, 18), url.pubdate
+    
+    url = Raev.url("http://kotaku.com/this-week-destiny-got-a-hell-of-a-lot-better-1706391634")
+    assert_equal_date Date.new(2015, 5, 23), url.pubdate
+    
+    url = Raev.url("http://www.rockpapershotgun.com/2014/07/03/beauty-beheld-future-unfolding/")
+    assert_equal_date Date.new(2014, 7, 3), url.pubdate
+    
+    url = Raev.url("http://jayisgames.com/review/the-black-forest-finding-friends.php")
+    assert_equal_date Date.new(2009, 12, 9), url.pubdate
+    
+    url = Raev.url("http://boingboing.net/2007/03/21/understanding-games.html")
+    assert_equal_date Date.new(2007, 3, 21), url.pubdate
+    
+    url = Raev.url("http://www.wired.com/2014/09/upcoming-a-gorgeous-adventure-game-that-mutates-for-each-player/")
+    assert_equal_date Date.new(2014, 9, 4), url.pubdate
+    
+    url = Raev.url("http://www.pcgamer.com/harebrained-schemes-hints-at-something-new-from-an-old-franchise/")
+    assert_equal_date Date.new(2015, 5, 21), url.pubdate    
+  end
+  
+  should "get author from url" do
+    url = Raev.url("http://www.rockpapershotgun.com/2014/07/03/beauty-beheld-future-unfolding/")
+    assert_equal "Adam Smith", url.author
+    
+    url = Raev.url("http://www.polygon.com/features/2013/3/25/4128022/gdc-gathering-of-game-makers")
+    assert_equal "Colin Campbell", url.author
+    
+    url = Raev.url("http://kotaku.com/worth-reading-some-kickstarters-are-lying-about-game-b-1706340013")
+    assert_equal "Patrick Klepek", url.author
+    
+    url = Raev.url("http://killscreendaily.com/articles/future-unfolding-wonder/")
+    assert_equal "Jess Joho", url.author
+    
+    url = Raev.url("http://www.creativeapplications.net/games/future-unfolding-procedurally-generated-world-both-beautiful-and-dangerous/")
+    assert_equal "Filip Visnjic", url.author
+  end
+  
+  private
+  
+  def assert_equal_date dateA, dateB
+    assert_equal dateA.strftime('%Y-%m-%d'), dateB.strftime('%Y-%m-%d')
+  end
+  
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: raev
 version: !ruby/object:Gem::Version
-  version: 0.1.12
+  version: 0.2.1
 platform: ruby
 authors:
 - Andreas Zecher
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-26 00:00:00.000000000 Z
+date: 2015-08-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.8.1
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -39,89 +53,89 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.1.1
 - !ruby/object:Gem::Dependency
-  name: shoulda
+  name: sanitize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 2.1.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: chronic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.3
-  type: :development
+        version: 0.9.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.3
+        version: 0.9.1
 - !ruby/object:Gem::Dependency
-  name: jeweler
+  name: shoulda
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: test-unit
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 1.7.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.5.4
+        version: 1.7.2
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: jeweler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.4
-  type: :runtime
+        version: 2.0.1
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: 1.4.4
+        version: 2.0.1
 - !ruby/object:Gem::Dependency
-  name: redirect_follower
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.1
-  type: :runtime
+        version: 2.5.4
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 2.5.4
 description: Fetch, parse and normalize meta data from websites.
 email: andreas@madebypixelate.com
 executables: []
@@ -167,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Fetch, parse and normalize meta data from websites.