feedalizer 0.1.0

data/CHANGELOG

@@ -0,0 +1,7 @@
+This changelog only mentions what you, as a user of Feedalizer,
+might want to know. See the Subversion log for more.
+
+[ 2006-08-23 20:23 ] 
+
+Added optional limit parameter to scrape_items.
+Thanks to Thanh Vinh Tang for the hint that it is needed.

data/COPYING

@@ -0,0 +1,18 @@
+Copyright (c) 2006 Christoffer Sawicki <christoffer.sawicki@gmail.com> 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+  
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+   
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,37 @@
+ ___           _      _ _
+| __|__ ___ __| |__ _| (_)______ _ _
+| _/ -_) -_) _` / _` | | |_ / -_) '_|
+|_|\___\___\__,_\__,_|_|_/__\___|_|
+
+  Project Website:
+  http://termos.vemod.net/feedalizer
+
+= Dependencies
+
+* Hpricot
+  http://code.whytheluckystiff.net/hpricot/
+  % gem install hpricot
+
+= Documentation
+
+First of all, see the included examples. Then read some Hpricot
+documentation (http://code.whytheluckystiff.net/hpricot/).
+
+(Yes, a tutorial would be nice.)
+
+= Copyright
+
+Copyright (c) Christoffer Sawicki <christoffer.sawicki@gmail.com> 2006
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License version 2 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA

data/TODO

@@ -0,0 +1,7 @@
+* Add more examples
+* Write some documentation (tutorial?)
+* Add note about CGI
+* Cache the retrieved HTML during script development?
+* Write unit test(s) for debug mode.
+* Add support for RSS 2.0
+* Add timeout control (a bit tricky to do nicely)

data/bin/feedalizer

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# Simple wrapper around Ruby :-)
+
+require "feedalizer"
+
+eval(ARGF.read)

data/examples/mongrel-news.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+# This script was contributed by Chu Yeow Cheah, thanks!
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'feedalizer'
+require 'time'
+
+url = 'http://mongrel.rubyforge.org/news.html'
+
+feedalize(url) do
+
+  feed.title = 'Mongrel News'
+  feed.about = '...'
+  feed.description = 'Latest Mongrel news from the official site'
+
+  scrape_items('h2') do |rss_item, html_element|
+
+    siblings = html_element.parent.containers
+    html_element_index = siblings.index(html_element)
+
+    date, title = html_element.innerHTML.strip.split(':')
+    date = Time.parse(date.sub(/-/, ' '))  # date is before the first ':'
+    title.strip!
+
+    # Grab siblings after the <h2> and put into description.
+    description = ''
+      (html_element_index+1...siblings.size).each do |i|
+      next_sibling = siblings[i]
+      unless 'h2' == next_sibling.stag.name
+        description = description + next_sibling.to_s
+      else
+        break
+      end
+    end
+
+    rss_item.link = url
+    rss_item.title = title
+    rss_item.date = date
+    rss_item.description = description
+  end
+
+  output!
+end

data/examples/pjvault.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+# This script was contributed by Christian Neukirchen, thanks!
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "feedalizer"
+require "time"
+
+feedalize("http://pjvault.com/news.html") do
+  feed.title = "Pearl Jam Vault"
+  feed.about = "..."
+  feed.description = "Pearl Jam News and Reviews"
+
+  scrape_items("li") do |rss_item, html_element|
+    link = html_element.search("a").first
+
+    rss_item.link  = link.attributes["href"]
+    rss_item.date  = Time.parse(html_element.to_s)
+    rss_item.title = link.children.last.to_s
+
+    rss_item.description = html_element.inner_html
+  end
+
+  output!
+end

data/examples/sydsvenskan-nemi.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "..", "lib")
+
+require "feedalizer"
+require "time"
+
+url = "http://sydsvenskan.se/serier/nemi/article101047.ece?context=serie"
+
+feedalize(url) do
+  feed.title = "Nemi"
+  feed.about = "..."
+  feed.description = "Daily Nemi strip scraped from Sydsvenskan"
+
+  scrape_items("option") do |rss_item, html_element|
+    rss_item.link  = html_element.attributes["value"]
+    rss_item.date  = Time.parse(html_element.inner_html)
+    rss_item.title = rss_item.date.strftime("%Y-%m-%d")
+
+    # This grabs the page for a particular strip and extracts the relevant img element
+    rss_item.description = grab_page(rss_item.link).search("//img[@width=748]")
+  end
+
+  output!
+end

data/lib/feedalizer.rb

@@ -0,0 +1,60 @@
+require "rss/maker"
+require "open-uri"
+require "hpricot"
+
+class Feedalizer
+  VERSION = "0.1.0"
+  
+  attr_reader :source
+
+  def initialize(url, &block)
+    @source = grab_page(url)
+    @rss = RSS::Maker::RSS10.new
+
+    feed.generator = "Feedalizer (http://termos.vemod.net/feedalizer)"
+    feed.link = url
+
+    instance_eval(&block) if block_given?
+
+    debug! if $DEBUG
+  end
+
+  def feed
+    @rss.channel
+  end
+
+  def scrape_items(hpricot_query, limit = 15)
+    elements = @source.search(hpricot_query)
+    
+    elements.first(limit).each do |element|
+      yield @rss.items.new_item, element
+    end
+  end
+
+  def grab_page(url)
+    open(url) { |io| Hpricot(io) }
+  end
+
+  # For backwards-compatibility
+  alias_method :parse_source, :grab_page
+
+  def output
+    @rss.to_rss.to_s unless $DEBUG
+  end
+
+  def output!(target = STDOUT)
+    target << output
+  end
+
+  def debug!
+    @rss.items.each do |item|
+      STDERR.puts [ item.title, item.date, item.link ].join("\n")
+      STDERR.puts
+    end
+  end
+end
+
+# A handy wrapper for Feedalizer.new :-)
+def feedalize(url, &block)
+  Feedalizer.new(url, &block)
+end

data/tests/tc_feedalizer.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require "test/unit"
+require "feedalizer"
+
+class TestFeedalizer < Test::Unit::TestCase
+  TEST_FILE = File.join(File.dirname(__FILE__), "test.html")
+
+  def setup
+    @feedalizer = Feedalizer.new(TEST_FILE)
+  end
+
+  def test_construction
+    assert_kind_of RSS::Maker::RSS10::Channel, @feedalizer.feed
+    assert_kind_of Hpricot::Doc, @feedalizer.source
+  end
+
+  def test_block
+    inside = nil
+    Feedalizer.new(TEST_FILE) { inside = self } rescue nil
+    assert_kind_of Feedalizer, inside
+  end
+
+  def test_default_feed_link
+    assert_equal TEST_FILE, @feedalizer.feed.link
+  end
+
+  def test_source
+    assert_equal "Test", @feedalizer.source.search("html/head/title").text
+  end
+
+  def test_scrape_items
+    elements = []
+
+    @feedalizer.scrape_items("div.item") do |item, element|
+      elements << element
+
+      assert_kind_of RSS::Maker::RSS10::Items::Item, item
+      assert_kind_of Hpricot::Elem, element
+    end
+
+    assert_equal 2, elements.size
+  end
+
+  def test_scrape_items_limit
+    elements = []
+
+    @feedalizer.scrape_items("div.item", 1) do |item, element|
+      elements << element
+    end
+
+    assert_equal 1, elements.size
+  end
+
+  def test_grab_page
+    assert_kind_of Hpricot::Doc, @feedalizer.grab_page(TEST_FILE)
+  end
+
+  def test_output
+    f = @feedalizer.feed
+    f.about = f.title = f.description = "..."
+
+    output = @feedalizer.output
+
+    assert output.include?('<rdf:RDF xmlns="http://purl.org/rss/1.0/"')
+    assert output.size > 600
+  end
+end

data/tests/test.html

@@ -0,0 +1,19 @@
+<html>
+  <head>
+    <title>Test</title>
+  </head>
+
+  <body>
+    <h1>News</h1>
+
+    <div class="item">
+      <h2>Foo</h2>
+      <p>Foo</p>
+    </div>
+
+    <div class="item">
+      <h2>Bar</h2>
+      <p>Bar</p>
+    </div>
+  </body>
+</html>

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: feedalizer
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-10-11 00:00:00 +02:00
+summary: Transforms web pages into RSS feeds
+require_paths: 
+- lib
+email: christoffer.sawicki@gmail.com
+homepage: http://termos.vemod.net/feedalizer
+rubyforge_project: feedalizer
+description: Feedalizer glues together Hpricot with Ruby's RSS library in a way that makes it dead easy to transform web pages into RSS feeds.
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+- Christoffer Sawicki
+files: 
+- CHANGELOG
+- lib/feedalizer.rb
+- tests/test.html
+- tests/tc_feedalizer.rb
+- TODO
+- COPYING
+- README
+- examples
+- examples/mongrel-news.rb
+- examples/pjvault.rb
+- examples/sydsvenskan-nemi.rb
+- bin/feedalizer
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: []
+
+executables: 
+- feedalizer
+extensions: []
+
+requirements: []
+
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Version::Requirement 
+    requirements: 
+    - - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+    version: