feed_ninja 0.0.4 → 0.0.7

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 33efbad5fc0d64ded0e3319aca0f866180264cf6
-  data.tar.gz: aa93bcb5af9786308807831e3494d2148aa783e4
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NWQyZDQ2ZGJmYThiNTMzOWVhMzY1MzhmM2IyOGU3MzllYzZhNGIwMw==
+  data.tar.gz: !binary |-
+    MjZhZDhhN2M3Yjg2N2VlYjllZGFjZTFiMjZlZTAzMmE2YTI2ZTZmNw==
 SHA512:
-  metadata.gz: e4f48c89a57c342f0c7c4689ba519c73cf4f23a1bfc4d389157506a10453278328688e7cbf2b9e4964b9d1f5a6fc872e2e52bbbfc2887b0ae4e7dce2324acc01
-  data.tar.gz: 8717dfcd4f7e361a0df1641420480003849ccbafe523971d078ab250d7156aaaddb5baa5c437e3ce4bb98e07528810592e4065b057ff0b112d147464099f42d9
+  metadata.gz: !binary |-
+    MmNkNDUzODJhZmIzMDI4MjAxYzY3MGIyY2I4MTIxNjYzOTA4ZGNkZjhlNzkx
+    M2U4MzRjMGE3ODU3Y2VkZjY2MDJiNTUyMTc1YjQ4OTYyYTQzYjQ4NDNmNGE3
+    OWEwY2FkOWU5OTEyMmQwYTcxNmIyMTE0ZmE4NDRhNGFhZGU2YmY=
+  data.tar.gz: !binary |-
+    N2U3NGMzMGFhOWU1YTY4ZDZmZGNlYzVkMmZiODZlNTlmZmYyYTVlNzA1NGQy
+    MTVhOTIyM2JmYTgxZjc2NjU3OTZmNzFhZjcxODEyNjIwZTliYzZkNjQ1OTAy
+    MDg5NzdlZDQ5YTFkODVlMmI2ZmEyNWRhMGY2YzU2MjliNDFlMjc=

data/lib/feed_ninja/atomish.rb

@@ -31,6 +31,10 @@ end
 class Entry
   attr_accessor :title, :link, :images, :updated, :summary, :id
 
+  def initialize
+    @summary = []
+  end
+
   def to_s
     %{  <entry>
     <title>#{@title}</title>
@@ -49,7 +53,6 @@ class Entry
         <img src="#{src}"/>
       </a>
       }
-    #end + summary || ""
-    end
+    end + @summary.join("\n")
   end
 end

data/lib/feed_ninja/extractor.rb

@@ -1,3 +1,6 @@
+require 'uri'
+require 'open-uri'
+
 class Extractor
   attr_accessor :doc
 
@@ -8,9 +11,11 @@ class Extractor
     end
   end
 
-  def extract_images(base_url, *xpaths)
-    Array(xpaths).collect_concat do |xpath|
-      extract_image(base_url, xpath)
+  def extract_images(base_url, xpaths)
+    LOGGER.debug{ "collecting images for #{xpaths}" }
+    [*xpaths].collect_concat do |xpath|
+      LOGGER.debug{ "collecting image:xpath #{xpath}" }
+      extract_image(URI(base_url), xpath)
     end
   end
 
@@ -19,14 +24,18 @@ class Extractor
       if(picture_src.to_s.start_with? 'http') then
         picture_src.to_s
       else
+        LOGGER.debug { "BASE URL IS #{base_url.class}" }
         "#{base_url.scheme}://#{base_url.host}/#{base_url.path}#{picture_src}"
       end
     end
   end
 
-  def extract_xml *xpaths
-    Array(xpaths).collect_concat do |xpath|
+  def extract_xml(xpaths)
+    LOGGER.debug{ "collecting text" }
+    [*xpaths].collect_concat do |xpath|
+      LOGGER.debug{ "collecting text:xpath #{xpath}" }
       @doc.xpath(xpath).collect do |result|
+        LOGGER.debug{ "collecting text:result #{result}" }
         result.to_s
       end
     end

data/lib/feed_ninja/feed_ninja.rb

@@ -5,6 +5,8 @@ require 'time'
 require 'thread'
 require 'thwait'
 
+Thread.abort_on_exception = true
+
 class FeedNinja
   attr_accessor :uri, :picture_xpath, :text_xpath, :title_regex, :limit
 
@@ -32,6 +34,9 @@ class FeedNinja
   # get the feed and iterate over the entries
   def fetch url
     open(url) do |feed|
+      if feed.content_encoding == ['gzip'] then
+        feed = Zlib::GzipReader.new(StringIO.new(feed.read)).read
+      end
       doc = RSS::Parser.parse(feed)
       initialize_writer(doc)
       process_items(doc)
@@ -52,6 +57,7 @@ class FeedNinja
 
   def process_item(original, feed_type, index)
     @writer.new_entry(index) do |entry|
+      LOGGER.debug{ "making new entry #{index}" }
       extractor = Extractor.new
       case feed_type
       when "atom"
@@ -68,9 +74,12 @@ class FeedNinja
         extractor.fetch original.link
       end
 
-      entry.images = extractor.extract_images @picture_xpath
+      LOGGER.debug{ "extracting for entry #{index} #{entry}" }
+      entry.images = extractor.extract_images(entry.link, @picture_xpath)
+      LOGGER.debug{ "RATATAT" }
       entry.summary = extractor.extract_xml @text_xpath
 
+      LOGGER.debug{ "adding entry #{index} #{entry}" }
       entry #it's kind of fishy to explicitly have to return the entry here...
     end
   end

data/lib/feed_ninja.rb

@@ -1,6 +1,7 @@
 require 'feed_ninja/feed_ninja'
 require 'feed_ninja/atomish'
 require 'feed_ninja/extractor'
+require 'logger'
 
 def get (url, &block)
   ninja = FeedNinja.new
@@ -9,3 +10,6 @@ def get (url, &block)
   puts "Content-type: application/atom+xml\n\n"
   puts ninja.to_s
 end
+
+LOGGER = Logger.new(STDERR)
+LOGGER.level = Logger::INFO

data/spec/feed_ninja_spec.rb

@@ -4,6 +4,8 @@ require 'feed_ninja'
 describe FeedNinja do
   before :each do
     @ninja = FeedNinja.new
+    @extractor = double()
+    Extractor.should_receive(:new).and_return(@extractor)
     #Extractor.stub(:extract_images => [])
     #Extractor.stub(:extract_xml => "")
   end
@@ -18,9 +20,9 @@ describe FeedNinja do
     @ninja.fetch 'spec/feeds/rss.xml'
   end
 
-  it 'should not read more than the given limit' do
+  it 'should not read more than the given limit', :focus do
     @ninja.limit = 1
-    Extractor.should_receive(:new).once
-    @ninja.fetch 'spec/feeds/rss.xml'
+    @extractor.should_receive(:fetch)
+    @ninja.fetch './spec/feeds/rss.xml'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feed_ninja
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.7
 platform: ruby
 authors:
 - Daniel Latzer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-16 00:00:00.000000000 Z
+date: 2014-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -38,9 +38,12 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.6.1
-description: |-
-  This gem can be used to take an RSS or Atom feed, follow the links they provide and extract images and/or text with xpath. The data is then reformatted into a new Atom feed.
-  It is inteded to be used with feeds that only provide a sneak peek of the content, to rip all the interesting bits out for displaying in your feed reader immediately.
+description: ! 'This gem can be used to take an RSS or Atom feed, follow the links
+  they provide and extract images and/or text with xpath. The data is then reformatted
+  into a new Atom feed.
+
+  It is inteded to be used with feeds that only provide a sneak peek of the content,
+  to rip all the interesting bits out for displaying in your feed reader immediately.'
 email: latzer.daniel@gmail.com
 executables: []
 extensions: []
@@ -68,12 +71,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []