harvester 0.8.0.pre.2 → 0.8.0.pre.3

data/README.rdoc

@@ -1,8 +1,8 @@
 = Harvester 0.8
 The Harvester is a web-based Feed-Aggregator
 
-A running instance can be seen at:
-http://astroblog.spaceboyz.net/harvester/
+Running instances can be seen at:
+http://blog-harvester.de or http://rubynetz.de
 
 The Harvester eats the feeds you want and produces a static html/feed page that aggregates all those.
 
@@ -14,7 +14,7 @@ Harvester 0.8 is alpha quality! There are still some unfixed bugs (e.g. database
 
 Install the harvester with
 
-  gem install harvester
+  gem install harvester --pre
 
 You can now create a new harvester project with
 
@@ -31,16 +31,6 @@ Then you will need to configure <tt>config.yaml</tt>, <tt>collections.yaml</tt>
 
 The <tt>collections.yaml</tt> file contains the links to your desired feeds.
 
-== Todo
-* Still some things broken after update
-* Fix database issues
-* Improve/fix feed parsing
-* Tidy up templates
-* Security issues
-* Optimize performance
-* Fix jabberbot
-* Implement some kind of tag filters
-
 == Usage
 
 Then run
@@ -55,13 +45,13 @@ This will create the output files. That's it ;).
 
 Usually, you want to run this command automatically every x seconds/minutes. You can use a program like <tt>cron</tt> or simply run
 
-  harvester clock
+  harvester clock <time>
 
 to start a simple scheduler.
 
 There some more harvester commands to explore. Run
 
-  harvester --help
+  harvester
 
 to get a command list.
 
@@ -69,6 +59,21 @@ to get a command list.
 
 ...
 
+
+== Todo
+* Still some things broken after update
+* Fix database issues
+* Fix wrong tweet escaping (sometimes: ä-->&auml;)
+* Improve/fix feed parsing (especially atom)
+* Fix chart generation
+* Tidy up templates / remove astro-specific links
+* Extract mrss.rb into extra gem
+
+* Security issues
+* Optimize performance
+* Fix jabber bot
+* Implement some kind of tag filters
+
 == Credits
 
 * {Astro}[https://github.com/astro] (2005-2008)

data/bin/harvester-create

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/create'
+Harvester.new_from_argv.create!

data/bin/harvester-jabber

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 # encoding: utf-8
 
-# TODO: pass differnet username as ARGV
+# TODO: pass different username as ARGV
 require_relative '../lib/harvester/jabber'
 Harvester.new_from_argv.jabber!

data/bin/harvester-maintenance

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/maintenance'
+Harvester.new_from_argv.maintenance!

data/bin/harvester-run

@@ -2,13 +2,16 @@
 # encoding: utf-8
 
 # run a complete harvester update :)
+require_relative '../lib/harvester'
 require_relative '../lib/harvester/fetch'
 require_relative '../lib/harvester/generate'
-require_relative '../lib/harvester/chart'
 require_relative '../lib/harvester/post'
 
 harve = Harvester.new_from_argv
+( require_relative '../lib/harvester/maintenance'
+harve.maintenance! ) if harve.settings['maintenance']
 harve.fetch!
 harve.generate!
-harve.chart! if harve.settings['chart']
+( require_relative '../lib/harvester/stats'
+harve.stats! ) if harve.settings['stats']
 harve.post!

data/bin/harvester-stats

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/stats'
+Harvester.new_from_argv.stats!

data/data/templates/index.html

@@ -240,8 +240,6 @@
             </div>
 	  </xsl:for-each>
 	</div>
-
-	<p style="display:none"><a href="http://cthulhu.c3d2.de/~astro/badpit.html">E-Mail Address Index</a></p>
       </body>
     </html>
   </xsl:template>

data/lib/harvester.rb

@@ -4,7 +4,7 @@ require 'yaml'
 require 'logger/colors'
 
 class Harvester
-  VERSION = '0.8.0.pre.2'
+  VERSION = '0.8.0.pre.3'
 
   attr_reader :config, :settings, :collections, :dbi, :logger
 
@@ -62,7 +62,7 @@ class Harvester
         database:          config['db']['database'],
         user:              config['db']['user'],
         password:          config['db']['password'],
-        host:              "localhost",
+        host:              "localhost", # FIXME?
     rescue Exception
       error 'Something is wrong with your database settings:'
       raise
@@ -81,7 +81,7 @@ COMMANDS:
     run          run a complete harvester update
     fetch        run only the fetch script
     generate     run only the generate script
-    chart        run only the generate chart script
+    stats        run only the generate chart script
     post         run only the post processing script
     db           start a database task (create or maintenance)
     clock        start the scheduler (cron replacement)
@@ -104,11 +104,11 @@ OPTIONS:} # automatically added as --help
       op.on('-p', '--post_script FILE') do |post_script|
         options['post_script'] = post_script
       end
-      op.on('-m', '--no-maintenance') do
-        options['no-maintenance'] = true
+      op.on('-m', '--maintenance') do
+        options['maintenance'] = true
       end
-      op.on('-c', '--chart') do
-        options['chart'] = true
+      op.on('-s', '--stats') do
+        options['stats'] = true
       end
     end.parse!
 
@@ -124,10 +124,18 @@ OPTIONS:} # automatically added as --help
   def error(msg) @logger.error(msg) end
   def fatal(msg) @logger.fatal(msg) end
 
-  # adds an info message before and after the block
-  def task(msg) # MAYBE: nested spaces+behaviour
+  def task(msg) # adds an info message before and after the block MAYBE: nested spaces+behaviour 
     info "[start] " + msg
     yield
     info "[done ] " + msg
   end
+
+  # database helpers
+  def sql_queries(task)
+    Dir[ File.dirname(__FILE__) + "/../data/sql/#{ @config['db']['driver'].downcase }/#{ task }*.sql" ].each
+  end
+
+  def sql_query(task)
+    File.dirname(__FILE__) + "/../data/sql/#{ @config['db']['driver'].downcase }/#{ task }.sql"
+  end
 end

data/lib/harvester/create.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+
+class Harvester
+  CREATE = true
+  # creates required database structure
+  def create!
+    info "CREATE"
+    task "create database tables" do
+      begin @dbi.transaction do
+        sql_queries(:create).each{ |sql|
+          info "* execute " + File.basename(sql)
+          @dbi.execute File.read(sql)
+        }
+      end; end
+    end
+  end
+end

data/lib/harvester/fetch.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 
 require_relative '../harvester'
-require_relative 'db'
 require_relative 'mrss'
 
 require 'eventmachine'
@@ -9,13 +8,74 @@ require 'em-http'
 require 'uri'
 
 class Harvester
-  module FETCH; end
-
+  FETCH = true
   # fetches new feed updates and store them in the database
   def fetch!
     info "FETCH"
-    maintenance! unless @settings['no-maintenance']
-    Fetcher.run @dbi, @collections, @settings, @logger do |*args| update(*args) end # results will be passed to the update function
+    Fetcher.run @dbi, @collections, @settings, @logger do |*args| update_db(*args) end # results will be passed to the update function
+  end
+
+  private
+
+  # saves result of a request in db
+  def update_db(rss_url, new_source, collection, response, rss_url_nice = rss_url)
+    rss = MRSS.parse(response)
+
+    begin @dbi.transaction do
+      # update source
+      if new_source
+        @dbi.execute "INSERT INTO sources (collection, rss, last, title, link, description) VALUES (?, ?, ?, ?, ?, ?)",
+          collection, rss_url, response['Last-Modified'], rss.title, rss.link, rss.description
+        info rss_url_nice + "Added as source"
+      else
+        @dbi.execute "UPDATE sources SET last=?, title=?, link=?, description=? WHERE collection=? AND rss=?",
+          response['Last-Modified'], rss.title, rss.link, rss.description, collection, rss_url
+        debug rss_url_nice + "Source updated"
+      end
+
+      # update items
+      items_new, items_updated = 0, 0
+      rss.items.each { |item|
+        description = item.description
+
+        # Link mangling
+        begin
+          link = URI::join((rss.link.to_s == '') ? URI.parse(rss_url).to_s : rss.link.to_s, item.link || rss.link).to_s
+        rescue URI::Error
+          link = item.link
+        end
+
+        # Push into database
+        db_title, = *@dbi.execute("SELECT title FROM items WHERE rss=? AND link=?", rss_url, link).fetch
+
+        if db_title.nil? || db_title.empty? # item is new
+          begin
+            @dbi.execute "INSERT INTO items (rss, title, link, date, description) VALUES (?, ?, ?, ?, ?)",
+              rss_url, item.title, link, item.date.to_s, description
+            items_new += 1
+          #rescue DBI::ProgrammingError
+          #  puts description
+          #  puts "#{$!.class}: #{$!}\n#{$!.backtrace.join("\n")}"
+          end
+        else
+          @dbi.execute "UPDATE items SET title=?, description=?, date=? WHERE rss=? AND link=?",
+            item.title, description, item.date.to_s, rss_url, link
+          items_updated += 1
+        end
+
+        # Remove all enclosures
+        @dbi.execute "DELETE FROM enclosures WHERE rss=? AND link=?", rss_url, link
+
+        # Re-add all enclosures
+        item.enclosures.each do |enclosure|
+          href = URI::join((rss.link.to_s == '') ? link.to_s : rss.link.to_s, enclosure['href']).to_s
+          @dbi.execute "INSERT INTO enclosures (rss, link, href, mime, title, length) VALUES (?, ?, ?, ?, ?, ?)",
+            rss_url, link, href, enclosure['type'], enclosure['title'],
+            !enclosure['length'] || enclosure['length'].empty? ? 0 : enclosure['length']
+        end
+      }
+      info rss_url_nice + "#{ items_new } new items, #{ items_updated } updated"
+    end; end
   end
 end
 
@@ -47,7 +107,7 @@ module Harvester::Fetcher
           db_rss, last = dbi.execute("SELECT rss, last FROM sources WHERE collection=? AND rss=?",
                           collection, rss_url).fetch
           new_source = db_rss.nil? || db_rss.empty?
-          uri = URI.parse rss_url
+          uri = URI.parse(rss_url)
 
           # prepare request
           header = {}
@@ -77,7 +137,7 @@ module Harvester::Fetcher
             elsif http.response.size > settings['size limit'].to_i
               logger.warn rss_url_nice + "Got too big repsonse: #{ response.size } bytes"
             else
-              yield rss_url, new_source, collection, http.response, rss_url_nice
+              yield rss_url, new_source, collection, http.response, rss_url_nice # TODO clean up
             end
 
             pending.delete rss_url # same url twice?
@@ -85,6 +145,7 @@ module Harvester::Fetcher
           end
         }
       }
+      EM.stop if pending.empty? # e.g. no collections configured
 
       EM.add_timer(settings['timeout'].to_i){
         pending.each { |rss_url| logger.warn rss_url_nice + 'Timed out' }

data/lib/harvester/generate.rb

@@ -14,13 +14,12 @@ rescue LoadError
 end
 
 class Harvester
-  module GENERATE; end
-
+  GENERATE = true
   # generates the static html/feed files
   def generate!
     info "GENERATE"
 
-    f        = Generator.new @dbi, @logger
+    f        = Generator.new @dbi, @settings, @logger
     xslt     = XML::XSLT.new
     xslt.xml = f.generate_root.to_s
 
@@ -54,9 +53,10 @@ end
 class Harvester::Generator
   FUNC_NAMESPACE = 'http://astroblog.spaceboyz.net/harvester/xslt-functions'
 
-  def initialize(dbi, logger)
-    @dbi = dbi
-    @logger = logger
+  def initialize(dbi, settings, logger) # TODO default values for arg2 and arg3
+    @dbi      = dbi
+    @settings = settings
+    @logger   = logger
     %w(collection-items feed-items item-description item-images item-enclosures).each { |func|
       XML::XSLT.extFunction(func, FUNC_NAMESPACE, self)
     }
@@ -80,7 +80,7 @@ class Harvester::Generator
     EntityTranslator.run(root, true, @logger)
   end
 
-  def collection_items(collection, max=23)
+  def collection_items(collection, max = 99)
     items = REXML::Element.new('items')
     @dbi.execute("SELECT items.title,items.date,items.link,items.rss FROM items,sources WHERE items.rss=sources.rss AND sources.collection LIKE ? ORDER BY items.date DESC LIMIT ?", collection, max.to_i).each{ |title,date,link,rss|
       if title # TODO: debug (sqlite)
@@ -95,7 +95,7 @@ class Harvester::Generator
     EntityTranslator.run(items, true, @logger)
   end
 
-  def feed_items(rss, max=23)
+  def feed_items(rss, max = 23)
     items = REXML::Element.new('items')
     @dbi.execute("SELECT title,date,link FROM items WHERE rss=? ORDER BY date DESC LIMIT ?", rss, max.to_i).each{ |title,date,link| #p rss,title,date,link
       # p title

data/lib/harvester/jabber.rb

@@ -1,5 +1,7 @@
 # encoding: ascii
 
+# FIXME: fix & integrate, maybe extra-gem?
+
 require_relative '../harvester'
 
 #require 'fastthread'

data/lib/harvester/maintenance.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+
+class Harvester
+  MAINTENANCE = true
+  # check for feed source changes
+  def maintenance!
+    info "MAINTENANCE"
+    task "look for sources to purge" do
+      purge = []
+      @dbi.execute("SELECT collection, rss FROM sources").each{ |dbc,dbr|
+        purge << [dbc, dbr] unless (@collections[dbc] || []).include? dbr
+      }
+
+      purge_rss = []
+      purge.each { |c,r|
+        info "* remove #{c}:#{r}..."
+        @dbi.execute "DELETE FROM sources WHERE collection=? AND rss=?", c, r
+        purge_rss << r
+      }
+
+      purge_rss.delete_if { |r|
+        purge_this = true
+
+        @collections.each { |cfc,cfr|
+          if purge_this
+            warn "* must keep #{r} because it's still in #{cfc}" if cfr && cfr.include?(r)
+            purge_this = !(cfr && cfr.include?(r))
+          end
+        }
+
+        !purge_this
+      }
+      purge_rss.each { |r|
+        info "* purge items from feed #{r}"
+        @dbi.execute "DELETE FROM items WHERE rss=?", r
+      }
+    end
+  end
+  alias purge! maintenance!
+end

data/lib/harvester/mrss.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 # Magic RSS
 # Helps getting around with RSS while ignoring standards and tolerating much
+# TODO: refactor out of harvester
 
 require 'rexml/document'
 require 'cgi'
@@ -187,6 +188,7 @@ class MRSS
         @e.s_text "link"
       end
       def description
+        @e.rss_content("itunes:summary") ||
         @e.rss_content("content:encoded") ||
           @e.rss_content("encoded") ||
           @e.rss_content("description")
@@ -291,9 +293,7 @@ class MRSS
     k = case xml.name
         when 'rss'
           RSS
-        when 'rdf'
-          RDF
-        when 'RDF'
+        when *%w[rdf RDF]
           RDF
         when 'feed'
           ATOM
@@ -303,6 +303,7 @@ class MRSS
     k.new(xml)
   end
 
+  # TODO also refactor out of mrss?
   module Util
     def self.detect_time(s)
       tz_offset = 0

data/lib/harvester/post.rb

@@ -3,11 +3,12 @@
 require_relative '../harvester'
 
 class Harvester
-  module POST; end
 
+  POST = true
   # runs the configured post processing scripts
   def post!(path = nil)
-    task "post process" do
+    info 'POST'
+    task 'post process' do
       if post_script = path || @config['settings']['post_script']
         error "Cannot find an executable script at #{ post_script }" unless test('x', post_script)
         exec post_script, @config['settings']['output']

data/lib/harvester/{chart.rb → stats.rb}

@@ -1,15 +1,13 @@
 # encoding: utf-8
 
 require_relative '../harvester'
-require_relative '../harvester/db'
 require 'gruff'
 
 class Harvester
-  module CHART; end
-
+  STATS = true
   # generates a fetch statistic image
-  def chart!
-    info "CHART"
+  def stats!
+    info "STATS"
     task "generate chart" do
       c = Chart::StatsPerCollection.new
       @dbi.execute( File.read( sql_query(:chart) ) ).each{ |date,collection|

metadata

@@ -2,7 +2,7 @@
 name: harvester
 version: !ruby/object:Gem::Version 
   prerelease: 6
-  version: 0.8.0.pre.2
+  version: 0.8.0.pre.3
 platform: ruby
 authors: 
 - astro
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-06 00:00:00 Z
+date: 2011-06-10 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdbi
@@ -131,39 +131,42 @@ email: mail@janlelis.de
 executables: 
 - harvester-new
 - harvester
+- harvester-create
 - harvester-clock
-- harvester-db
-- harvester-chart
+- harvester-stats
 - harvester-run
 - harvester-generate
 - harvester-fetch
 - harvester-jabber
 - harvester-post
+- harvester-maintenance
 extensions: []
 
 extra_rdoc_files: []
 
 files: 
+- lib/harvester/maintenance.rb
 - lib/harvester/generate.rb
 - lib/harvester/generator/entity_translator.rb
 - lib/harvester/generator/link_absolutizer.rb
-- lib/harvester/db.rb
-- lib/harvester/chart.rb
+- lib/harvester/create.rb
 - lib/harvester/post.rb
+- lib/harvester/stats.rb
 - lib/harvester/jabber.rb
 - lib/harvester/fetch.rb
 - lib/harvester/mrss.rb
 - lib/harvester.rb
 - bin/harvester-new
 - bin/harvester
+- bin/harvester-create
 - bin/harvester-clock
-- bin/harvester-db
-- bin/harvester-chart
+- bin/harvester-stats
 - bin/harvester-run
 - bin/harvester-generate
 - bin/harvester-fetch
 - bin/harvester-jabber
 - bin/harvester-post
+- bin/harvester-maintenance
 - README.rdoc
 - CHANGELOG.rdoc
 - data/ent/HTMLsymbol.ent

data/bin/harvester-chart

@@ -1,5 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-require_relative '../lib/harvester/chart'
-Harvester.new_from_argv.chart!

data/bin/harvester-db

@@ -1,15 +0,0 @@
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-require_relative '../lib/harvester/db'
-harve = Harvester.new_from_argv
-
-case action = ARGV.shift
-when 'create'
-  harve.create!
-when 'maintenance'
-  harve.maintenance!
-else
-  puts '[DB] Do nothing...'
-end
-  

data/lib/harvester/db.rb

@@ -1,123 +0,0 @@
-# encoding: utf-8
-
-require_relative '../harvester'
-
-class Harvester
-  module DB; end
-
-  # creates required database structure
-  def create!
-    task "create database tables" do
-      begin @dbi.transaction do
-        sql_queries(:create).each{ |sql|
-          info "* execute " + File.basename(sql)
-          @dbi.execute File.read(sql)
-        }
-      end; end
-    end
-  end
-
-  # check for feed source changes
-  def maintenance!
-    task "look for sources to purge" do
-      purge = []
-      @dbi.execute("SELECT collection, rss FROM sources").each{ |dbc,dbr|
-        purge << [dbc, dbr] unless (@collections[dbc] || []).include? dbr
-      }
-
-      purge_rss = []
-      purge.each { |c,r|
-        info "* remove #{c}:#{r}..."
-        @dbi.execute "DELETE FROM sources WHERE collection=? AND rss=?", c, r
-        purge_rss << r
-      }
-
-      purge_rss.delete_if { |r|
-        purge_this = true
-
-        @collections.each { |cfc,cfr|
-          if purge_this
-            warn "* must keep #{r} because it's still in #{cfc}" if cfr && cfr.include?(r)
-            purge_this = !(cfr && cfr.include?(r))
-          end
-        }
-
-        !purge_this
-      }
-      purge_rss.each { |r|
-        info "* purge items from feed #{r}"
-        @dbi.execute "DELETE FROM items WHERE rss=?", r
-      }
-    end
-  end
-
-  private
-
-  def update(rss_url, new_source, collection, response, rss_url_nice = rss_url)
-    rss = MRSS.parse(response)
-
-    begin @dbi.transaction do
-      # update source
-      if new_source
-        @dbi.execute "INSERT INTO sources (collection, rss, last, title, link, description) VALUES (?, ?, ?, ?, ?, ?)",
-          collection, rss_url, response['Last-Modified'], rss.title, rss.link, rss.description
-        info rss_url_nice + "Added as source"
-      else
-        @dbi.execute "UPDATE sources SET last=?, title=?, link=?, description=? WHERE collection=? AND rss=?",
-          response['Last-Modified'], rss.title, rss.link, rss.description, collection, rss_url
-        debug rss_url_nice + "Source updated"
-      end
-
-      # update items
-      items_new, items_updated = 0, 0
-      rss.items.each { |item|
-        description = item.description
-
-        # Link mangling
-        begin
-          link = URI::join((rss.link.to_s == '') ? uri.to_s : rss.link.to_s, item.link || rss.link).to_s
-        rescue URI::Error
-          link = item.link
-        end
-
-        # Push into database
-        db_title, = *@dbi.execute("SELECT title FROM items WHERE rss=? AND link=?", rss_url, link).fetch
-
-        if db_title.nil? || db_title.empty? # item is new
-          begin
-            @dbi.execute "INSERT INTO items (rss, title, link, date, description) VALUES (?, ?, ?, ?, ?)",
-              rss_url, item.title, link, item.date.to_s, description
-            items_new += 1
-          #rescue DBI::ProgrammingError
-          #  puts description
-          #  puts "#{$!.class}: #{$!}\n#{$!.backtrace.join("\n")}"
-          end
-        else
-          @dbi.execute "UPDATE items SET title=?, description=?, date=? WHERE rss=? AND link=?",
-            item.title, description, item.date.to_s, rss_url, link
-          items_updated += 1
-        end
-
-        # Remove all enclosures
-        @dbi.execute "DELETE FROM enclosures WHERE rss=? AND link=?", rss_url, link
-
-        # Re-add all enclosures
-        item.enclosures.each do |enclosure|
-          href = URI::join((rss.link.to_s == '') ? link.to_s : rss.link.to_s, enclosure['href']).to_s
-          @dbi.execute "INSERT INTO enclosures (rss, link, href, mime, title, length) VALUES (?, ?, ?, ?, ?, ?)",
-            rss_url, link, href, enclosure['type'], enclosure['title'],
-            !enclosure['length'] || enclosure['length'].empty? ? 0 : enclosure['length']
-        end
-      }
-      info rss_url_nice + "#{ items_new } new items, #{ items_updated } updated"
-    end; end
-  end
-
-  def sql_queries(task)
-    Dir[ File.dirname(__FILE__) + "/../../data/sql/#{ @config['db']['driver'].downcase }/#{ task }*.sql" ].each
-  end
-
-  def sql_query(task)
-    File.dirname(__FILE__) + "/../../data/sql/#{ @config['db']['driver'].downcase }/#{ task }.sql"
-  end
-end