feed_ninja 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e232c6d96bcbdfda50a9985ab6430c2275dc1608
+  data.tar.gz: 2a0d5f1d965290ba077c0bb7eceb176d47a53f55
+SHA512:
+  metadata.gz: 0d3c349beea7ef6835ff680fbc5974f569f2e260fffbc42c55a6b61874a5a5236bc4a1bc6a14311341a2a80982048901daac31e42e0f24a87c25bfeeeb0bc9ab
+  data.tar.gz: 75e0af1da786f34fcd13c7969c7500c49b00299dcc5d76a1a9ed995bf716d4f5fa858aee035bf584d6b30dbdab1baab5369e7039f52edaa268ca25090593c731

data/README.md

@@ -0,0 +1,12 @@
+#FeedNinja
+This gem can be used to take an RSS or Atom feed, follow the links they provide and extract images and/or text with xpath. The data is then reformatted into a new Atom feed.
+It is inteded to be used with feeds that only provide a sneak peek of the content, to rip all the interesting bits out for displaying in your feed reader immediately.
+
+##Example Usage
+  require 'feed_ninja'
+
+  get 'http://example.com/rss' do
+    picture_at '//foo/img/@src'
+    text_at '//bar/span'
+    title_matches /^News/
+  end

data/lib/feed_ninja/atomish.rb

@@ -0,0 +1,55 @@
+class AtomIshWriter
+  attr_accessor :title, :link, :updated
+  def initialize
+    @entries = []
+  end
+
+  def new_entry
+    item = Entry.new
+    item = yield item
+    @entries << item;
+  end
+
+  def to_s
+    %{<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>#{@title}</title>
+  <id>#{@link}</id>
+  <link href="#{@link}"/>
+  <updated>#{@updated}</updated>
+  <author>
+    <name>FeedNinja</name>
+    <uri>http://github.com/Tourniquet/feedninja</uri>
+    <email>latzer.daniel@gmail.com</email>
+  </author>
+    #{@entries.inject { |memo, entry| memo.to_s + entry.to_s }.to_s}</feed>}
+
+  end
+end
+
+class Entry
+  attr_accessor :title, :link, :images, :updated, :summary, :id
+
+  def to_s
+    %{  <entry>
+    <title>#{@title}</title>
+    <link rel="alternate" type="text/html" href="#{@link}" />
+    <id>#{@id}</id>
+    <updated>#{@updated}</updated>
+    <content type="html">#{self.content.encode(:xml => :text)}</content>
+  </entry>
+}
+  end
+
+  def content
+    Array(@images).inject("") do |memo, src|
+      memo += %{
+      <a href="#{src}">
+        <img src="#{src}"/>
+      </a>
+      }
+    #end + summary || ""
+    end
+  end
+end

data/lib/feed_ninja/extractor.rb

@@ -0,0 +1,34 @@
+class Extractor
+  attr_accessor :doc
+
+  def fetch uri
+    open(uri) do |site|
+      @doc = Nokogiri::HTML(site)
+      #return extract_image(doc, site.base_uri), extract_xml(doc)
+    end
+  end
+
+  def extract_images(base_url, *xpaths)
+    Array(xpaths).collect_concat do |xpath|
+      extract_image(base_url, xpath)
+    end
+  end
+
+  def extract_image(base_url, xpath)
+    @doc.xpath(xpath).collect do | picture_src |
+      if(picture_src.to_s.start_with? 'http') then
+        picture_src.to_s
+      else
+        "#{base_url.scheme}://#{base_url.host}/#{base_url.path}#{picture_src}"
+      end
+    end
+  end
+
+  def extract_xml *xpaths
+    Array(xpaths).collect_concat do |xpath|
+      @doc.xpath(xpath).collect do |result|
+        result.to_s
+      end
+    end
+  end
+end

data/lib/feed_ninja/feed_ninja.rb

@@ -0,0 +1,95 @@
+require 'rss'
+require 'open-uri'
+require 'nokogiri'
+require 'time'
+
+class FeedNinja
+  attr_accessor :uri, :picture_xpath, :text_xpath, :title_regex, :limit
+  attr_accessor :extractor
+
+  def initialize
+    @limit = 2
+    @extractor = Extractor.new
+    @writer = AtomIshWriter.new
+    @ninja_prefix = "N! "
+  end
+
+  def initialize_writer doc
+    @writer.updated = DateTime.now.to_s
+
+    case doc.feed_type
+    when "atom"
+      @writer.title = @ninja_prefix + doc.title.content
+      @writer.link = doc.link.href
+    when "rss"
+      @writer.title = @ninja_prefix + doc.channel.title
+      @writer.link = doc.channel.link
+    else
+      raise "Invalid feed format"
+    end
+  end
+
+  # get the feed and iterate over the entries
+  def fetch url
+    open(url) do |feed|
+      doc = RSS::Parser.parse(feed)
+      initialize_writer(doc)
+      process_items(doc)
+    end
+  end
+
+  def process_items doc
+    items = doc.items
+    if title_regex
+      items = items.select { |item| title_regex =~ item.title }
+    end
+    items.first(@limit).each do |item|
+
+      #TODO add multithreading here; be sure to use multiple extractor instances
+      process_item item, doc.feed_type
+    end
+  end
+
+  def process_item original, feed_type
+    @writer.new_entry do |entry|
+      case feed_type
+      when "atom"
+        entry.title = original.title.content
+        entry.link = original.link.href
+        entry.updated = original.updated
+        entry.id = original.id
+        @extractor.fetch original.link.href
+      when "rss"
+        entry.title = original.title
+        entry.link = original.link
+        entry.updated = original.pubDate ? original.pubDate.xmlschema : DateTime.now.to_s
+        entry.id = entry.link
+        @extractor.fetch original.link
+      end
+
+      entry.images = @extractor.extract_images @picture_xpath
+      entry.summary = @extractor.extract_xml @text_xpath
+
+      entry #it's kind of fishy to explicitly have to return the entry here...
+    end
+  end
+
+  def to_s
+    @writer.to_s
+  end
+
+  ## DSL convenience setters
+
+  def picture_at *xpath
+    @picture_xpath = xpath
+  end
+
+  def text_at *xpath
+    @text_xpath = xpath
+  end
+
+  def title_matches regex
+    @title_regex = regex
+  end
+end
+

data/lib/feed_ninja.rb

@@ -0,0 +1,12 @@
+require 'feed_ninja/feed_ninja'
+require 'feed_ninja/atomish'
+require 'feed_ninja/extractor'
+
+def get (url, &block)
+  ninja = FeedNinja.new
+  ninja.instance_eval(&block)
+  ninja.fetch(url)
+
+  puts "Content-type: application/atom+xml\n"
+  puts ninja.to_s
+end

data/spec/atomish_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+require 'feed_ninja'
+
+describe AtomIshWriter do
+  it 'should output a valid atom feed' do
+    writer = AtomIshWriter.new
+    writer.title = 'test'
+    writer.link = 'http://example.com/atom'
+    writer.updated = DateTime.now.to_s
+
+    writer.new_entry do |entry|
+      entry = Entry.new
+      entry.title = "title"
+      entry.link = "http://example.com/one"
+      entry.id = entry.link
+      entry.images = ["http://example.com/one.jpg", "http://example.com/two.jpg"]
+      entry.summary = "First part of the story"
+      entry.updated = DateTime.now.to_s
+    end
+
+    writer.new_entry do |entry|
+      entry = Entry.new
+      entry.title = "title"
+      entry.link = "http://example.com/two"
+      entry.id = entry.link
+      entry.images = ["http://example.com/one.jpg", "http://example.com/two.jpg"]
+      entry.summary = "Second part of the story"
+      entry.updated = (DateTime.now - 60).to_s
+    end
+
+    RSS::Parser.parse(writer.to_s)
+  end
+end

data/spec/extractor_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'feed_ninja'
+
+describe Extractor do
+  before :each do
+    @extractor = Extractor.new
+    @extractor.fetch 'spec/pages/one.html'
+    @base = URI('http://example.com')
+  end
+
+  it 'should extract one image with relative url' do
+    xpath = "//div[@id='one_image_relative']/img/@src"
+    picture = @extractor.extract_images(@base, xpath)
+
+    picture.should == ["http://example.com/one.jpg"]
+  end
+
+  it 'should extract one image with absolute url' do
+    xpath = "//div[@id='one_image_absolute']/img/@src"
+    base = URI('http://wrong.com') #base URI shouldn't be applied here
+    picture = @extractor.extract_images(base, xpath)
+
+    picture.should == ["http://example.com/one.jpg"]
+  end
+
+  it 'should extract several images' do
+    xpath = "//div[@id='several_images']/img/@src"
+    pictures = @extractor.extract_images(@base, xpath)
+
+    pictures.size.should == 2
+    pictures.should == ["http://example.com/one.jpg", "http://example.com/two.jpg"]
+  end
+
+  it 'should extract some paragraphs' do
+    paragraphs = @extractor.extract_xml "//div[@id='paragraphs']/p"
+
+    paragraphs.should == %w{<p>one</p> <p>two</p> <p>three</p>}
+  end
+end

data/spec/feed_ninja_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'feed_ninja'
+
+describe FeedNinja do
+  before :each do
+    @ninja = FeedNinja.new
+    @extractor = double()
+    @ninja.extractor = @extractor
+    @extractor.stub(:extract_images)
+    @extractor.stub(:extract_xml)
+  end
+
+  it 'should read an atom feed' do
+    @extractor.should_receive(:fetch).twice
+    @ninja.fetch 'spec/feeds/atom.xml'
+  end
+
+  it 'should read an RSS feed' do
+    @extractor.should_receive(:fetch).twice
+    @ninja.fetch 'spec/feeds/rss.xml'
+  end
+
+  it 'should not read more than the given limit' do
+    @ninja.limit = 1
+    @extractor.should_receive(:fetch).once
+    @ninja.fetch 'spec/feeds/rss.xml'
+  end
+end

data/spec/feeds/atom.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+
+  <title>atom</title>
+  <link rel="alternate" type="text/html" href="http://example.com/atom"/>
+  <updated>2007-07-13T18:30:02Z</updated>
+  <author>
+    <name>feedninja</name>
+  </author>
+  <id>http://example.com/atom</id>
+
+  <entry>
+    <title>one</title>
+    <link href="http://example.com/one"/>
+    <id>1</id>
+    <updated>2007-07-13T18:30:02Z</updated>
+    <summary>summary_one</summary>
+  </entry>
+
+  <entry>
+    <title>two</title>
+    <link href="http://example.com/two"/>
+    <id>2</id>
+    <updated>2007-07-13T18:30:02Z</updated>
+    <summary>summary_two</summary>
+  </entry>
+
+</feed>

data/spec/feeds/rss.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<rss version="2.0">
+
+<channel>
+  <title>rss_feed</title>
+  <link>http://www.example.com</link>
+  <description>descri </description>
+  <item>
+    <title>one</title>
+    <link>http://example.com/one</link>
+    <description>description_one</description>
+  </item>
+  <item>
+    <title>two</title>
+    <link>http://example.com/two</link>
+    <description>description_two</description>
+  </item>
+</channel>
+</rss>

data/spec/pages/one.html

@@ -0,0 +1,18 @@
+<html>
+  <div id="one_image_relative">
+    <img src="one.jpg"/>
+  </div>
+  <div id="one_image_absolute">
+    <img src="http://example.com/one.jpg"/>
+  </div>
+  <div id="several_images">
+    <img src="one.jpg"/>
+    <img src="two.jpg"/>
+  </div>
+  <div id="paragraphs">
+    <p>one</p>
+    <p>two</p>
+    <span>combo_breaker</span>
+    <p>three</p>
+  </div>
+</html>

data/spec/spec_helper.rb

@@ -0,0 +1,17 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: feed_ninja
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Daniel Latzer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.14.1
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+description: |-
+  This gem can be used to take an RSS or Atom feed, follow the links they provide and extract images and/or text with xpath. The data is then reformatted into a new Atom feed.
+  It is inteded to be used with feeds that only provide a sneak peek of the content, to rip all the interesting bits out for displaying in your feed reader immediately.
+email: latzer.daniel@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/feed_ninja.rb
+- lib/feed_ninja/atomish.rb
+- lib/feed_ninja/extractor.rb
+- lib/feed_ninja/feed_ninja.rb
+- spec/atomish_spec.rb
+- spec/extractor_spec.rb
+- spec/feed_ninja_spec.rb
+- spec/feeds/atom.xml
+- spec/feeds/rss.xml
+- spec/pages/one.html
+- spec/spec_helper.rb
+homepage: http://github.com/tourniquet/feedninja
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A tiny helper to rip the interesting bits out of RSS and Atom feeds
+test_files: []