RubyGems - feed_searcher - Versions diffs - 0.0.4 → 0.0.5 - Mend

feed_searcher 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (12) hide show

data/.travis.yml +6 -0
data/README.md +18 -0
data/Rakefile +4 -0
data/feed_searcher.gemspec +3 -0
data/lib/feed_searcher/page.rb +71 -3
data/lib/feed_searcher/version.rb +1 -1
data/spec/feed_searcher_spec.rb +102 -41
data/spec/fixtures/example.atom +15 -0
data/spec/fixtures/example.html +18 -0
data/spec/fixtures/example.rss +12 -0
data/spec/spec_helper.rb +3 -0
metadata +57 -2

data/.travis.yml ADDED Viewed

@@ -0,0 +1,6 @@
+rvm:
+- 1.9.3
+- 2.0.0
+branches:
+  only:
+    - master

data/README.md CHANGED Viewed

@@ -1,6 +1,8 @@
 # FeedSearcher
 Search RSS feed URLs from the given URL.
+[![Build Status](https://travis-ci.org/fastladder/feed_searcher.png?branch=master)](https://travis-ci.org/fastladder/feed_searcher)
 ## Installation
 ```
@@ -14,3 +16,19 @@ require "feed_searcher"
 FeedSearcher.search("https://github.com/r7kamura/feed_searcher")
 #=> ["https://github.com/r7kamura/feed_searcher/commits/master.atom"]
 ```
+## Internal
+Let me explain how FeedSearcher works along its execution sequence.
+1. Fetches the HTML source of the given URL
+2. Finds link elements (represented as XPath format)
+3. Extracts URLs from the elements via its `href` attribute
+4. Includes the given URL if its resource itself is a feed
+5. Converts to relative path to absolute path
+FeedSearcher finds link elements matcing following XPath patterns.
+* //link[@rel='alternate'][@type='application/atom+xml']
+* //link[@rel='alternate'][@type='application/rdf+xml']
+* //link[@rel='alternate'][@type='application/rss+xml']

data/Rakefile CHANGED Viewed

@@ -1 +1,5 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+task :default => :spec

data/feed_searcher.gemspec CHANGED Viewed

@@ -20,8 +20,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "mechanize", ">= 1.0.0"
   spec.add_dependency "nokogiri"
+  spec.add_development_dependency "activesupport"
   spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", ">= 2.13.0"
+  spec.add_development_dependency "simplecov"
   spec.add_development_dependency "webmock"
 end

data/lib/feed_searcher/page.rb CHANGED Viewed

@@ -1,5 +1,11 @@
 class FeedSearcher
   class Page
+    EXTENSIONS = %w[
+      atom
+      rdf
+      rss
+    ]
     MIME_TYPES = %w[
       application/atom+xml
       application/rdf+xml
@@ -13,12 +19,54 @@ class FeedSearcher
     end
     def feed_urls
-      feed_attributes.map {|attribute| attribute["href"] }
+      urls = []
+      urls << url if (has_feed_mime_type? || has_feed_extension?) && xml?
+      urls += links.map {|link| link["href"] }
     end
     private
-    def feed_attributes
+    def has_xml_declaration?
+      !!body.index("<?xml")
+    end
+    def has_feed_mime_type?
+      MIME_TYPES.include?(mime_type)
+    end
+    def has_feed_extension?
+      EXTENSIONS.include?(extension)
+    end
+    def parsable_as_xml?
+      !!xml
+    end
+    def xml?
+      has_xml_declaration? && parsable_as_xml?
+    end
+    def url
+      page.uri.to_s
+    end
+    def content_type
+      page.response["content-type"]
+    end
+    def mime_type
+      content_type.sub(/;.*\z/, "") if content_type
+    end
+    def extension
+      File.extname(page.uri.path).sub(/^\./, "")
+    end
+    def body
+      page.body
+    end
+    def links
       root.xpath("//link[@rel='alternate' and (#{types_query})]")
     end
@@ -27,7 +75,27 @@ class FeedSearcher
     end
     def root
-      Nokogiri.HTML(page.body)
+      xml || html
+    end
+    def xml
+      if @xml.nil?
+        @xml = parse_xml
+      else
+        @xml
+      end
+    end
+    def html
+      Nokogiri.HTML(body)
+    end
+    def parse_xml
+      Nokogiri.XML(body) do |config|
+        config.options = Nokogiri::XML::ParseOptions::STRICT | Nokogiri::XML::ParseOptions::NOENT
+      end
+    rescue
+      false
     end
   end
 end

data/lib/feed_searcher/version.rb CHANGED Viewed

@@ -1,3 +1,3 @@
 class FeedSearcher
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/spec/feed_searcher_spec.rb CHANGED Viewed

@@ -1,50 +1,111 @@
 require "spec_helper"
+require "active_support/core_ext/string/strip"
 describe FeedSearcher do
   describe ".search" do
-    before do
-      stub_request(:get, "http://example.com/").to_return(
-        :body => <<-EOF
-          <!DOCTYPE HTML>
-          <html>
-            <head>
-              <meta charset="UTF-8">
-              <link href="http://example.com/1" rel="alternate" type="application/atom+xml" />
-              <link href="http://example.com/2" rel="alternate" type="application/rdf+xml" />
-              <link href="http://example.com/3" rel="alternate" type="application/rss+xml" />
-              <link href="http://example.com/4" rel="alternate" type="application/xml" />
-              <link href="http://example.com/5" rel="resource"  type="application/rss+xml" />
-              <link href="http://www.example.com/6" rel="alternate"  type="application/rss+xml" />
-              <link href="http://other-example.com/7" rel="alternate"  type="application/rss+xml" />
-              <link href="/8" rel="alternate" type="application/rss+xml" />
-            </head>
-            <body>
-              body
-            </body>
-          </html>
-        EOF
-      )
+    context "when the specified resource is HTML" do
+      before do
+        stub_request(:get, "http://example.com/").to_return(
+          :body => <<-EOS.strip_heredoc
+            <!DOCTYPE HTML>
+            <html>
+              <head>
+                <meta charset="UTF-8">
+                <link href="http://example.com/1" rel="alternate" type="application/atom+xml" />
+                <link href="http://example.com/2" rel="alternate" type="application/rdf+xml" />
+                <link href="http://example.com/3" rel="alternate" type="application/rss+xml" />
+                <link href="http://example.com/4" rel="alternate" type="application/xml" />
+                <link href="http://example.com/5" rel="resource"  type="application/rss+xml" />
+                <link href="http://www.example.com/6" rel="alternate"  type="application/rss+xml" />
+                <link href="http://other-example.com/7" rel="alternate"  type="application/rss+xml" />
+                <link href="/8" rel="alternate" type="application/rss+xml" />
+              </head>
+              <body>
+                body
+              </body>
+            </html>
+          EOS
+        )
+      end
+      # This example makes sure the following specifications.
+      #
+      #   * it recognizes application/atom+xml
+      #   * it recognizes application/rdf+xml
+      #   * it recognizes application/rss+xml
+      #   * it does not recognize application/xml
+      #   * it keeps subdomain
+      #   * it keeps other domain
+      #   * it converts relative path to absolute url
+      #
+      it "returns feed URLs from link elements in the specified resource" do
+        FeedSearcher.search("http://example.com/").should == %w[
+          http://example.com/1
+          http://example.com/2
+          http://example.com/3
+          http://www.example.com/6
+          http://other-example.com/7
+          http://example.com/8
+        ]
+      end
     end
-    # This example makes sure the following specifications.
-    #
-    #   * it recognizes application/atom+xml
-    #   * it recognizes application/rdf+xml
-    #   * it recognizes application/rss+xml
-    #   * it does not recognizes application/xml
-    #   * it keeps subdomain
-    #   * it keeps other domain
-    #   * it converts absolute url
-    #
-    it "returns feed URLs from given URL" do
-      FeedSearcher.search("http://example.com/").should == %w[
-        http://example.com/1
-        http://example.com/2
-        http://example.com/3
-        http://www.example.com/6
-        http://other-example.com/7
-        http://example.com/8
-      ]
+    context "when the specified resource has feed MIME type and be parsable as XML" do
+      before do
+        stub_request(:get, "http://example.com/").to_return(
+          :headers => { "Content-Type" => "application/rss+xml; charset=UTF-8" },
+          :body    => <<-EOS.strip_heredoc
+            <?xml version="1.0" encoding="UTF-8"?>
+            <rss>
+              <channel>
+                <title>title</title>
+                <link>http://exmple.com/</link>
+                <item>
+                  <title>item title</title>
+                  <link>http://example.com/item</link>
+                  <description>item description</description>
+                </item>
+              </channel>
+            </rss>
+          EOS
+        )
+      end
+      it "returns itself as a feed url" do
+        FeedSearcher.search("http://example.com/").should == %w[
+          http://example.com/
+        ]
+      end
+    end
+    context "when the specified resource has feed extension and be parsable as XML" do
+      before do
+        stub_request(:get, "http://example.com/feed.atom").to_return(
+          :body => <<-EOS.strip_heredoc
+            <?xml version="1.0" encoding="UTF-8"?>
+            <feed xmlns="http://www.w3.org/2005/Atom">
+              <title>title</title>
+              <link rel="self" href="http://example.com/1"/>
+              <link rel="alternate" href="http://example.com/"/>
+              <entry>
+                <title>item title</title>
+                <link rel="alternate" href="http://example.com/"/>
+                <content type="html">
+                  <div xmlns="http://www.w3.org/1999/xhtml">
+                    <p>item content</p>
+                  </div>
+                </content>
+              </entry>
+            </feed>
+          EOS
+        )
+      end
+      it "returns itself as a feed url" do
+        FeedSearcher.search("http://example.com/feed.atom").should == %w[
+          http://example.com/feed.atom
+        ]
+      end
     end
   end
 end

data/spec/fixtures/example.atom ADDED Viewed

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title>title</title>
+  <link rel="self" href="http://example.com/1"/>
+  <link rel="alternate" href="http://example.com/"/>
+  <entry>
+    <title>item title</title>
+    <link rel="alternate" href="http://example.com/"/>
+    <content type="html">
+      <div xmlns="http://www.w3.org/1999/xhtml">
+        <p>item content</p>
+      </div>
+    </content>
+  </entry>
+</feed>

data/spec/fixtures/example.html ADDED Viewed

@@ -0,0 +1,18 @@
+<!DOCTYPE HTML>
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <link href="http://example.com/1" rel="alternate" type="application/atom+xml" />
+    <link href="http://example.com/2" rel="alternate" type="application/rdf+xml" />
+    <link href="http://example.com/3" rel="alternate" type="application/rss+xml" />
+    <link href="http://example.com/4" rel="alternate" type="application/xml" />
+    <link href="http://example.com/5" rel="resource"  type="application/rss+xml" />
+    <link href="http://www.example.com/6" rel="alternate"  type="application/rss+xml" />
+    <link href="http://other-example.com/7" rel="alternate"  type="application/rss+xml" />
+    <link href="/8" rel="alternate" type="application/rss+xml" />
+  </head>
+  <body>
+    body
+  </body>
+ </html>

data/spec/fixtures/example.rss ADDED Viewed

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss>
+  <channel>
+    <title>title</title>
+    <link>http://exmple.com/</link>
+    <item>
+      <title>item title</title>
+      <link>http://example.com/item</link>
+      <description>item description</description>
+    </item>
+  </channel>
+</rss>

data/spec/spec_helper.rb CHANGED Viewed

@@ -1,3 +1,6 @@
+require "simplecov"
+SimpleCov.start
 $LOAD_PATH.unshift File.expand_path("../../lib/feed_searcher", __FILE__)
 require "feed_searcher"
 require "webmock/rspec"

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feed_searcher
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease:
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-03-17 00:00:00.000000000 Z
+date: 2013-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -59,6 +75,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -91,6 +123,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 2.13.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -115,6 +163,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -125,6 +174,9 @@ files:
 - lib/feed_searcher/page.rb
 - lib/feed_searcher/version.rb
 - spec/feed_searcher_spec.rb
+- spec/fixtures/example.atom
+- spec/fixtures/example.html
+- spec/fixtures/example.rss
 - spec/spec_helper.rb
 homepage: https://github.com/r7kamura/feed_searcher
 licenses:
@@ -153,5 +205,8 @@ specification_version: 3
 summary: Search RSS feed URLs from the given URL
 test_files:
 - spec/feed_searcher_spec.rb
+- spec/fixtures/example.atom
+- spec/fixtures/example.html
+- spec/fixtures/example.rss
 - spec/spec_helper.rb
 has_rdoc: