harvester 0.8.0.pre.1

data/lib/harvester/mrss.rb

@@ -0,0 +1,355 @@
+# encoding: utf-8
+# Magic RSS
+# Helps getting around with RSS while ignoring standards and tolerating much
+
+require 'rexml/document'
+require 'cgi'
+require 'iconv'
+
+# Search macros
+class REXML::Element
+  def s(expr)
+    each_element(expr) { |e| return e }
+    nil
+  end
+
+  def s_text(expr)
+    if (e = s(expr))
+      (e.cdatas.size > 0) ? e.cdatas.to_s : e.text
+    else
+      nil
+    end
+  end
+
+  def s_attr(expr, attr)
+    (e = s(expr)) ? e.attributes[attr] : nil
+  end
+
+  def rss_content(expr) # TODO: understand & refactor
+    if e = s(expr)
+      CGI::unescapeHTML(e.children.join)
+    end
+  end
+
+  def atom_content(expr)
+    e = s(expr)
+    if e
+      type = e.attributes['type']
+      mode = e.attributes['mode']
+
+      if type == 'xhtml' or mode == 'xml'
+        e.children.join
+      elsif type == 'html' or mode == 'escaped'
+        if e.cdatas.size > 0
+          e.cdatas.to_s
+        else
+          e.text
+        end
+      else # 'text'
+        if e.cdatas.size > 0
+          CGI::escapeHTML(e.cdatas.to_s)
+        else
+          CGI::escapeHTML(e.text.to_s)
+        end
+      end
+    else
+      nil
+    end
+  end
+
+  def atom_content_text(expr)
+    c = atom_content(expr)
+    if c
+      c.gsub!(/<.+?>/, '')
+      CGI::unescapeHTML(c)
+    else
+      nil
+    end
+  end
+
+  def atom_link
+    link, rank = 0, 0
+
+    each_element('link') do |e|
+      rel = e.attributes['rel']
+      type = e.attributes['type']
+
+      e_rank = (rel == 'alternate' ? 2 : 1) *
+        (rel == 'enclosure' ? 0 : 1) *
+        ((type == 'text/html' ? 2 : 0) +
+         (type == 'application/xhtml+xml' ? 1 : 0) +
+         1)
+
+      if e_rank > rank
+        link = e.attributes['href']
+        rank = e_rank
+      end
+    end
+
+    link
+  end
+end
+
+
+class MRSS
+  class AbstractMethod < RuntimeError; end
+
+  def initialize(root)
+    @root = root
+  end
+
+  # Common properties
+  def items
+    res = []
+
+    @root.each_element(items_expr) do |i|
+      item = self.class::Item.new(i)
+
+      res << item unless item.title.to_s == '' and item.link.to_s == ''
+    end
+
+    res
+  end
+
+  # Abstract stuff
+  def title
+    raise AbstractMethod.new
+  end
+
+  def type
+    raise AbstractMethod.new
+  end
+
+  def link
+    raise AbstractMethod.new
+  end
+
+  def description
+    raise AbstractMethod.new
+  end
+
+  def feed_expr
+    raise AbstractMethod.new
+  end
+
+  def items_expr
+    raise AbstractMethod.new
+  end
+
+  # Item
+
+  class Item
+    def initialize(e)
+      @e = e
+    end
+    def title
+      raise AbstractMethod.new
+    end
+    def link
+      raise AbstractMethod.new
+    end
+    def description
+      raise AbstractMethod.new
+    end
+    def date
+      raise AbstractMethod.new
+    end
+    def enclosures
+      raise AbstractMethod.new
+    end
+  end
+
+  # Flavours
+
+  # RSS 0.9 & 2.0
+  class RSS < MRSS
+    def title
+      @root.s_text "#{feed_expr}/title"
+    end
+    def feed_expr
+      "/rss/channel"
+    end
+    def items_expr
+      "/rss/channel/item"
+    end
+    def link
+      @root.s_text "#{feed_expr}/link"
+    end
+    def description
+      @root.s_text "#{feed_expr}/description"
+    end
+
+    class Item < ::MRSS::Item
+      def title
+        @e.s_text "title"
+      end
+      def link
+        @e.s_text "link"
+      end
+      def description
+        @e.rss_content("content:encoded") ||
+          @e.rss_content("encoded") ||
+          @e.rss_content("description")
+      end
+      def date
+        d = @e.s_text('date') || @e.s_text('pubDate') || @e.s_text('dc:date')
+        MRSS::Util.detect_time(d).localtime
+      end
+      def enclosures
+        r = []
+        @e.each_element('enclosure') do |e|
+          h = {
+            'href' => e.attributes['url'],
+            'type' => e.attributes['type'],
+            'title' => e.attributes['title'],
+            'length' => e.attributes['length']
+          }
+          r << h
+        end
+        r
+      end
+    end
+  end
+
+  # RSS 1.0
+  class RDF < RSS
+    def feed_expr
+      "/*/channel"
+    end
+    def items_expr
+      "/*/item"
+    end
+  end
+
+  # ATOM 0.3 & 1.0
+  class ATOM < MRSS
+    def feed_expr
+      "/feed"
+    end
+    def items_expr
+      "/feed/entry"
+    end
+    def title
+      @root.atom_content_text "title"
+    end
+    def link
+      @root.atom_link
+    end
+    def description
+      @root.atom_content_text('tagline') ||
+        @root.atom_content_text('subtitle')
+    end
+
+    class Item < ::MRSS::Item
+      def title
+        @e.atom_content_text "title"
+      end
+      def link
+        @e.atom_link
+      end
+      def description
+        @e.atom_content("content") ||
+          @e.atom_content("summary")
+      end
+      def date
+        d = @e.s_text('published') || @e.s_text('issued') || @e.s_text('updated')
+        MRSS::Util.detect_time(d).localtime
+      end
+      def enclosures
+        r = []
+        @e.each_element("link") do |e|
+          if e.attributes['rel'] == 'enclosure'
+            h = {
+              'href' => e.attributes['href'],
+              'type' => e.attributes['type'],
+              'title' => e.attributes['title'],
+              'length' => e.attributes['length']
+            }
+            r << h
+          end
+        end
+        r
+      end
+    end
+  end
+
+
+  def self.parse(body)
+    begin
+      xml = REXML::Document.new(body).root
+    rescue REXML::ParseException
+      xml = REXML::Document.new(Iconv::iconv('UTF-8', 'ISO-8859-15', body).to_s).root
+    end
+
+    begin
+      # Try to validate UTF-8
+      Iconv::iconv('UTF-8', 'UTF-8', xml.to_s).to_s
+    rescue Iconv::IllegalSequence, Iconv::InvalidCharacter
+      xml = REXML::Document.new(Iconv::iconv('UTF-8', 'ISO-8859-15', body).to_s).root
+    end
+
+    k = case xml.name
+        when 'rss'
+          RSS
+        when 'rdf'
+          RDF
+        when 'RDF'
+          RDF
+        when 'feed'
+          ATOM
+        else
+          raise "Unknwon feed format: #{xml.name} (#{xml.namespace})"
+        end
+    k.new(xml)
+  end
+
+  module Util
+    def self.detect_time(s)
+      tz_offset = 0
+
+      # Fix it up
+      s.strip!
+
+      s.scan(/([\+\-])(\d\d):?(\d\d)$/).each do |plus_minus,hours,minutes|
+        tz_offset = ((hours.to_i * 60) + minutes.to_i) * 60
+        tz_offset *= (plus_minus == '+') ? -1 : 1
+      end
+
+      # 2004-04-01T21:23+00:00
+      # 2004-09-05T21:23Z
+      s.scan(/^(\d+)-(\d+)-(\d+)T(\d+):(\d+)/).each do |y,mo,d,h,m|
+        return Time.gm(y.to_i, mo.to_i, d.to_i, h.to_i, m.to_i) + tz_offset
+      end
+
+      # 2004-04-01T21:23:23+00:00
+      s.scan(/^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)\+(\d+):(\d+)$/).each do |y,mo,d,h,m,s,tz_h,tz_m|
+        return Time.gm(y.to_i, mo.to_i, d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
+      end
+
+      # Wed, 20 Apr 2005 19:38:15 +0200
+      # Fri, 22 Apr 2005 10:31:12 GMT
+      months = {"Jan" => 1, "Feb" => 2, "Mar" => 3, "Mär" => 3, "Apr" => 4, "May" => 5, "Mai" => 5, "Jun" => 6, "Jul" => 7, "Aug" => 8, "Sep" => 9, "Oct" => 10, "Okt" => 10, "Nov" => 11, "Dec" => 12, "Dez" => 12 }
+      s.scan(/^(.+?), +(\d+) (.+?) (\d+) (\d+):(\d+):(\d+) /).each do |wday,d,mo,y,h,m,s|
+        return Time.gm(y.to_i, months[mo], d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
+      end
+
+      # 06 May 2007 02:20:00
+      s.scan(/^(\d+) (.+?) (\d{4}) (\d\d):(\d\d):(\d\d)/).each do |d,mo,y,h,m,s|
+        return Time.gm(y.to_i, months[mo], d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
+      end
+
+      # Mon Apr 25 00:00:00 +0000 2011
+      s.scan(/^(.+?) (.+?) (\d+) (\d+):(\d+):(\d+) .* (\d{4})/).each do |wday,mo,d,h,m,s,y|
+        return Time.gm(y.to_i, months[mo], d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
+      end
+
+      # 2011-05-27
+      s.scan(/^(\d{4})-(\d\d)-(\d\d)/).each do |y,mo,d|
+        return Time.gm(y.to_i, months[mo], d.to_i) + tz_offset
+      end
+
+      # ugly <now> fallback
+      Time.new
+    end
+  end
+end

data/lib/harvester/post.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+
+class Harvester
+  module POST; end
+
+  # runs the configured post processing scripts
+  def post!(path = nil)
+    task "post process" do
+      if post_script = path || @config['settings']['post_script']
+        error "Cannot find an executable script at #{ post_script }" unless test('x', post_script)
+        exec post_script, @config['settings']['output']
+      else
+        warn 'No post processing script configured or passed as argument!'
+      end
+    end#task
+  end
+end

metadata

@@ -0,0 +1,237 @@
+--- !ruby/object:Gem::Specification 
+name: harvester
+version: !ruby/object:Gem::Version 
+  prerelease: 6
+  version: 0.8.0.pre.1
+platform: ruby
+authors: 
+- astro
+- Neingeist
+- Tigion
+- Josef Spillner
+- Jan Lelis
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-27 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rdbi
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rdbi-driver-sqlite3
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: logger-colors
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: ruby-xslt
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: rmagick
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: gruff
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: clockwork
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id010
+description: The harvester collects your favourite feeds and generates static html/feed pages
+email: mail@janlelis.de
+executables: 
+- harvester-new
+- harvester
+- harvester-clock
+- harvester-db
+- harvester-chart
+- harvester-run
+- harvester-generate
+- harvester-fetch
+- harvester-jabber
+- harvester-post
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/harvester/generate.rb
+- lib/harvester/generator/entity_translator.rb
+- lib/harvester/generator/link_absolutizer.rb
+- lib/harvester/db.rb
+- lib/harvester/chart.rb
+- lib/harvester/post.rb
+- lib/harvester/jabber.rb
+- lib/harvester/fetch.rb
+- lib/harvester/mrss.rb
+- lib/harvester.rb
+- bin/harvester-new
+- bin/harvester
+- bin/harvester-clock
+- bin/harvester-db
+- bin/harvester-chart
+- bin/harvester-run
+- bin/harvester-generate
+- bin/harvester-fetch
+- bin/harvester-jabber
+- bin/harvester-post
+- README.rdoc
+- CHANGELOG.rdoc
+- data/ent/HTMLsymbol.ent
+- data/ent/HTMLspecial.ent
+- data/ent/HTMLlat1.ent
+- data/templates/rss.rdf
+- data/templates/atom.xml
+- data/templates/index.html
+- data/templates/static/harvester_lte_ie6.css
+- data/templates/static/harvester_ie7.css
+- data/templates/static/harvester.css
+- data/templates/static/harvester.gif
+- data/templates/rss-all.rdf
+- data/templates/atom-all.xml
+- data/sql/sqlite3/create.table.jabbersettings.sql
+- data/sql/sqlite3/create.table.items.sql
+- data/sql/sqlite3/create.table.jabbersubscriptions.sql
+- data/sql/sqlite3/create.table.sources.sql
+- data/sql/sqlite3/create.view.last48hours.sql
+- data/sql/sqlite3/chart.sql
+- data/sql/sqlite3/create.table.enclosures.sql
+- data/sql/harvester-0.6-mysql.diff
+- data/sql/dbd-mysql-isotime.diff
+- data/sql/harvester-0.7-mysql.diff
+- data/sql/postgresql/create.table.jabbersettings.sql
+- data/sql/postgresql/create.table.items.sql
+- data/sql/postgresql/create.table.jabbersubscriptions.sql
+- data/sql/postgresql/create.table.sources.sql
+- data/sql/postgresql/create.view.last48hours.sql
+- data/sql/postgresql/chart.sql
+- data/sql/postgresql/create.table.enclosures.sql
+- data/sql/mysql/create.table.jabbersettings.sql
+- data/sql/mysql/create.table.items.sql
+- data/sql/mysql/create.table.jabbersubscriptions.sql
+- data/sql/mysql/create.table.sources.sql
+- data/sql/mysql/create.view.last48hours.sql
+- data/sql/mysql/chart.sql
+- data/sql/mysql/create.table.enclosures.sql
+- collections.yaml
+- config.yaml
+- Rakefile
+- harvester.gemspec
+homepage: https://github.com/janlelis/harvester
+licenses: 
+- AGPL
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.1
+signing_key: 
+specification_version: 3
+summary: Web-based feed aggregator
+test_files: []
+