harvester 0.8.0.pre.1 → 0.8.0.pre.2

data/README.rdoc

@@ -6,6 +6,10 @@ http://astroblog.spaceboyz.net/harvester/
 
 The Harvester eats the feeds you want and produces a static html/feed page that aggregates all those.
 
+== Warning
+
+Harvester 0.8 is alpha quality! There are still some unfixed bugs (e.g. database stuff)
+
 == Installation
 
 Install the harvester with
@@ -29,6 +33,7 @@ The <tt>collections.yaml</tt> file contains the links to your desired feeds.
 
 == Todo
 * Still some things broken after update
+* Fix database issues
 * Improve/fix feed parsing
 * Tidy up templates
 * Security issues

data/bin/harvester-run

@@ -10,5 +10,5 @@ require_relative '../lib/harvester/post'
 harve = Harvester.new_from_argv
 harve.fetch!
 harve.generate!
-harve.chart! unless harve.settings['no-chart']
+harve.chart! if harve.settings['chart']
 harve.post!

data/data/sql/mysql/create.view.last48hours.sql

@@ -1 +1 @@
-create view last48hrs as select items.rss, items.title, items.link, sources.title as blogtitle, sources.collection from items, sources where items.rss = sources.rss and now() - interval '48 hour' < items.date order by date;
+create view last48hrs as select items.rss, items.title, items.link, sources.title as blogtitle, sources.collection from items, sources where items.rss = sources.rss and now() - interval 48 hour < items.`date` order by date;

data/harvester.gemspec

@@ -14,14 +14,14 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = ">= 1.3.6"
   # main
   s.add_dependency 'rdbi'
-  s.add_dependency 'rdbi-driver-sqlite3'
+  s.add_dependency 'rdbi-driver-sqlite3' # BUGGY oO
   s.add_dependency 'logger-colors'
   # fetch
   s.add_dependency 'eventmachine'
   s.add_dependency 'em-http-request'
   # generate
   s.add_dependency 'ruby-xslt'
-  s.add_dependency 'hpricot'
+  s.add_dependency 'nokogiri'
   # chart
   s.add_dependency 'rmagick'
   s.add_dependency 'gruff'

data/lib/harvester.rb

@@ -4,7 +4,7 @@ require 'yaml'
 require 'logger/colors'
 
 class Harvester
-  VERSION = '0.8.0.pre.1'
+  VERSION = '0.8.0.pre.2'
 
   attr_reader :config, :settings, :collections, :dbi, :logger
 
@@ -61,7 +61,8 @@ class Harvester
       @dbi = RDBI::connect config['db']['driver'],
         database:          config['db']['database'],
         user:              config['db']['user'],
-        password:          config['db']['password']
+        password:          config['db']['password'],
+        host:              "localhost",
     rescue Exception
       error 'Something is wrong with your database settings:'
       raise
@@ -106,8 +107,8 @@ OPTIONS:} # automatically added as --help
       op.on('-m', '--no-maintenance') do
         options['no-maintenance'] = true
       end
-      op.on('-s', '--no-chart') do
-        options['no-chart'] = true
+      op.on('-c', '--chart') do
+        options['chart'] = true
       end
     end.parse!
 

data/lib/harvester/db.rb

@@ -93,8 +93,8 @@ class Harvester
           #  puts "#{$!.class}: #{$!}\n#{$!.backtrace.join("\n")}"
           end
         else
-          @dbi.execute "UPDATE items SET title=?, description=? WHERE rss=? AND link=?",
-            item.title, description, rss_url, link
+          @dbi.execute "UPDATE items SET title=?, description=?, date=? WHERE rss=? AND link=?",
+            item.title, description, item.date.to_s, rss_url, link
           items_updated += 1
         end
 

data/lib/harvester/generate.rb

@@ -86,7 +86,7 @@ class Harvester::Generator
       if title # TODO: debug (sqlite)
         item = items.add(REXML::Element.new('item'))
         item.add(REXML::Element.new('title')).text = title
-        item.add(REXML::Element.new('date')).text = Time.parse(date).xmlschema
+        item.add(REXML::Element.new('date')).text = Time.parse(date.to_s).xmlschema
         item.add(REXML::Element.new('link')).text = link
         item.add(REXML::Element.new('rss')).text = rss
       end
@@ -102,7 +102,7 @@ class Harvester::Generator
       if title # TODO: debug (sqlite)
         item = items.add(REXML::Element.new('item'))
         item.add(REXML::Element.new('title')).text = title
-        item.add(REXML::Element.new('date')).text = Time.parse(date).xmlschema
+        item.add(REXML::Element.new('date')).text = Time.parse(date.to_s).xmlschema
         item.add(REXML::Element.new('link')).text = link
       end
     }

data/lib/harvester/generator/entity_translator.rb

@@ -4,7 +4,7 @@ class Harvester; class Generator; end; end
 # This module translates old-fashioned entities into utf-8
 class Harvester::Generator::EntityTranslator
   def self.run(doc, with_xmldecl = true, logger = nil)
-    logger ||= Logger.new(STDOUT)
+    @logger = logger || Logger.new(STDOUT)
 
     @entities = {}
     %w(HTMLlat1.ent HTMLsymbol.ent HTMLspecial.ent).each do |file|

data/lib/harvester/generator/link_absolutizer.rb

@@ -5,35 +5,29 @@ class Harvester; class Generator; end; end
 module Harvester::Generator::LinkAbsolutizer
   def self.run(body, base, logger = nil)
     logger ||= Logger.new(STDOUT)
-    require 'hpricot'
+    require 'nokogiri'
+    require 'uri'
 
-    html = Hpricot("<html><body>#{body}</body></html>")
-    (html/'a').each { |a|
-      begin
-        f = a.get_attribute('href')
-        t = URI::join(base, f.to_s).to_s
-        logger.debug "* rewriting #{f.inspect} => #{t.inspect}" if f != t
-        a.set_attribute('href', t)
-      rescue URI::Error
-        logger.debug "* cannot rewrite relative URL: #{a.get_attribute('href').inspect}" unless a.get_attribute('href') =~ /^[a-z]{2,10}:/
-      end
+    html = Nokogiri::HTML("<html><body>#{body}</body></html>")
+    [%w[img src], %w[a href]].each{ |elem, attr|
+      html.css(elem).each{ |e|
+        begin
+          src = e[attr]
+          uri = URI::join(base, src.to_s).to_s
+          if src.to_s != uri.to_s
+            logger.debug "* rewriting #{src.inspect} => #{uri.inspect}" 
+            e[attr] = uri.to_s
+          end
+        rescue URI::Error
+          logger.debug "* cannot rewrite relative URL: #{src.inspect}" #unless src.to_s =~ /^[a-z]{2,10}:/
+        end
+      }
     }
-    (html/'img').each { |img|
-      begin
-        f = img.get_attribute('src')
-        t = URI::join(base, f.to_s).to_s
-        logger.debug "* rewriting #{f.inspect} => #{t.inspect}" if f != t
-        img.set_attribute('src', t)
-      rescue URI::Error
-        logger.debug "* cannot rewrite relative URL: #{img.get_attribute('href').inspect}" unless img.get_attribute('href') =~ /^[a-z]{2,10}:/
-      end
-    }
-    html.search('/html/body/*').to_s
-  rescue Hpricot::Error => e
-    logger.error "Hpricot::Error: #{e}"
-    body
+    html.css('body').children.to_s
   rescue LoadError
-    logger.warn "* hpricot not found, will not mangle relative links in <description/>"
+    logger.warn "* nokogiri not found, will not mangle relative links in <description/>"
     body
+  rescue Exception => e
+    logger.warn "* there was a nokogiri exception: #{e}"
   end
 end

data/lib/harvester/mrss.rb

@@ -343,6 +343,11 @@ class MRSS
         return Time.gm(y.to_i, months[mo], d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
       end
 
+      # 2011-05-27 17:46:28
+      s.scan(/^(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)/).each do |y,mo,d,h,m,s|
+        return Time.gm(y.to_i, months[mo], d.to_i, h.to_i, m.to_i, s.to_i) + tz_offset
+      end
+
       # 2011-05-27
       s.scan(/^(\d{4})-(\d\d)-(\d\d)/).each do |y,mo,d|
         return Time.gm(y.to_i, months[mo], d.to_i) + tz_offset

metadata

@@ -2,7 +2,7 @@
 name: harvester
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 0.8.0.pre.1
+  version: 0.8.0.pre.2
 platform: ruby
 authors: 
 - astro
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-27 00:00:00 Z
+date: 2011-06-06 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdbi
@@ -83,7 +83,7 @@ dependencies:
   type: :runtime
   version_requirements: *id006
 - !ruby/object:Gem::Dependency 
-  name: hpricot
+  name: nokogiri
   prerelease: false
   requirement: &id007 !ruby/object:Gem::Requirement 
     none: false