harvester 0.8.0.pre.1

data/data/templates/static/harvester_ie7.css

@@ -0,0 +1,15 @@
+/*
+ * harvester_ie7.css
+ *
+ * Description: Anpassungen des Defaultstylesheets 'harvester.css'
+                für IE7
+ * 
+ */
+
+/* ------------------------------------------------------ */
+/* Blogeintraege */
+/* ------------------------------------------------------ */
+
+.entry-content pre, .entry-content code {
+  overflow: scroll;
+}

data/data/templates/static/harvester_lte_ie6.css

@@ -0,0 +1,27 @@
+/*
+ * harvester_lte_ie6.css
+ *
+ * Description: Anpassungen des Defaultstylesheets 'harvester.css'
+                für IE6 und aelter Versionen
+ * 
+ */
+
+/* ------------------------------------------------------ */
+/* Blogeintraege */
+/* ------------------------------------------------------ */
+
+.entry {
+  width: 100%;
+  /*width: 98%;*/ /* dirty hack */
+}
+
+.entry-content pre,
+.entry-content code {
+  width: 90%; /* dirty hack */
+  overflow: scroll;
+}
+
+.tickers3 .ticker ul li { 
+  padding-bottom: 1em;
+  width: 100%;
+}

data/harvester.gemspec

@@ -0,0 +1,35 @@
+# encoding: utf-8
+require 'rubygems' unless defined? Gem
+require File.dirname(__FILE__) + "/lib/harvester"
+ 
+Gem::Specification.new do |s|
+  s.name        = "harvester"
+  s.version     = Harvester::VERSION
+  s.authors     = ["astro", "Neingeist", "Tigion", "Josef Spillner", "Jan Lelis"]
+  s.email       = "mail@janlelis.de"
+  s.homepage    = "https://github.com/janlelis/harvester"
+  s.summary     = "Web-based feed aggregator"
+  s.description = "The harvester collects your favourite feeds and generates static html/feed pages"
+  s.required_ruby_version = ">= 1.9.2"
+  s.required_rubygems_version = ">= 1.3.6"
+  # main
+  s.add_dependency 'rdbi'
+  s.add_dependency 'rdbi-driver-sqlite3'
+  s.add_dependency 'logger-colors'
+  # fetch
+  s.add_dependency 'eventmachine'
+  s.add_dependency 'em-http-request'
+  # generate
+  s.add_dependency 'ruby-xslt'
+  s.add_dependency 'hpricot'
+  # chart
+  s.add_dependency 'rmagick'
+  s.add_dependency 'gruff'
+  # clock
+  s.add_dependency 'clockwork'
+  # jabber
+  # s.add_dependency 'xmpp4r'
+  s.files = Dir.glob(%w|lib/**/*.rb bin/* [A-Z]*.{txt,rdoc} data/**/* *.yaml|) + %w|Rakefile harvester.gemspec|
+  s.executables = Dir['bin/*'].map{|f| File.basename f }
+  s.license = "AGPL"
+end

data/lib/harvester.rb

@@ -0,0 +1,132 @@
+require 'rubygems' unless defined? Gem
+require 'rdbi'
+require 'yaml'
+require 'logger/colors'
+
+class Harvester
+  VERSION = '0.8.0.pre.1'
+
+  attr_reader :config, :settings, :collections, :dbi, :logger
+
+  # takes an options string (to which overwrites settings) and creates a new Harvester instance
+  def initialize(options = {})
+    # command line options
+    options['config'] ||= './config.yaml'
+
+    # load config
+    begin
+      config_path = File.expand_path( options.delete('config') )
+      @config = YAML::load_file config_path
+    rescue Errno::ENOENT
+      raise LoadError, "Could not find a yaml config file at #{ config_path }"
+    end
+
+    # instance variable helpers
+    @settings = {
+      'collections' => 'collections.yaml',
+      'timeout'     => 90,
+      'size limit'  => 200_000,
+      'log_level'   => Logger::DEBUG, # 0
+      'log_file'    => STDOUT,
+    }
+    @settings.merge! @config['settings']
+    @settings.merge! options
+
+    # load collections
+    begin
+      @collections = YAML::load_file @settings['collections']
+    rescue Errno::ENOENT
+      raise LoadError, "Could not find a yaml collections file at #{ File.expand_path( options[:config] ) }"
+    end
+
+    # init logger
+    @logger = Logger.new(
+      if !@settings['log_file'] || @settings['log_file'] =~ /^STD(?:OUT|ERR)$/
+        Object.const_get(@settings['log_file'])
+      else
+        @settings['log_file']
+      end
+    )
+    @logger.formatter = proc { |level, datetime, appname, msg| "#{msg}\n" }
+    @logger.level = if %w[debug info warn error fatal].include?(@settings['log_level'].to_s.downcase)
+      Logger::Severity.const_get(@settings['log_level'].to_s.upcase)
+    else
+      @settings['log_level'].to_i
+    end
+
+    # connect to db
+    begin
+      require 'rdbi/driver/' + config['db']['driver'].downcase # FIXME?
+
+      @dbi = RDBI::connect config['db']['driver'],
+        database:          config['db']['database'],
+        user:              config['db']['user'],
+        password:          config['db']['password']
+    rescue Exception
+      error 'Something is wrong with your database settings:'
+      raise
+    end
+  end
+
+  # creates a new harvester using the command-line options to configure it
+  def self.new_from_argv
+    options = {}
+
+    require 'optparse'
+    OptionParser.new do |op|
+      op.banner = %q{USAGE:
+    harvester <COMMAND> [OPTIONS]
+COMMANDS:
+    run          run a complete harvester update
+    fetch        run only the fetch script
+    generate     run only the generate script
+    chart        run only the generate chart script
+    post         run only the post processing script
+    db           start a database task (create or maintenance)
+    clock        start the scheduler (cron replacement)
+    new          create a new harvester project
+    jabber       start the jabber bot (not implemented yet)
+OPTIONS:} # automatically added as --help
+      op.on('-v', '--version') do
+        puts Harvester::VERSION
+        exit
+      end
+      op.on('-c', '--config FILE') do |config|
+        options['config'] = config
+      end
+      op.on('-l', '--log_file FILE') do |log_file|
+        options['log_file'] = log_file
+      end
+      op.on('-L', '--log_level NUMBER') do |log_level|
+        options['log_level'] = log_level
+      end
+      op.on('-p', '--post_script FILE') do |post_script|
+        options['post_script'] = post_script
+      end
+      op.on('-m', '--no-maintenance') do
+        options['no-maintenance'] = true
+      end
+      op.on('-s', '--no-chart') do
+        options['no-chart'] = true
+      end
+    end.parse!
+
+    Harvester.new options
+  end
+
+  protected
+
+  # logger helpers
+  def debug(msg) @logger.debug(msg) end
+  def info(msg)  @logger.info(msg)  end
+  def warn(msg)  @logger.warn(msg)  end
+  def error(msg) @logger.error(msg) end
+  def fatal(msg) @logger.fatal(msg) end
+
+  # adds an info message before and after the block
+  def task(msg) # MAYBE: nested spaces+behaviour
+    info "[start] " + msg
+    yield
+    info "[done ] " + msg
+  end
+end

data/lib/harvester/chart.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+require_relative '../harvester/db'
+require 'gruff'
+
+class Harvester
+  module CHART; end
+
+  # generates a fetch statistic image
+  def chart!
+    info "CHART"
+    task "generate chart" do
+      c = Chart::StatsPerCollection.new
+      @dbi.execute( File.read( sql_query(:chart) ) ).each{ |date,collection|
+        c.add_one(collection, Date.parse(date).day)
+      }
+      Chart.new(c).write File.join( @config['settings']['output'], '/chart.jpg' )
+    end
+  end
+end
+
+# generates a fetch statistics image using gruff
+class Harvester::Chart
+  def initialize(stats, options = {}) # TODO configure g with options
+    @g = Gruff::Line.new(300)
+    @g.title = "Harvested items per day"
+    @g.x_axis_label = "Days"
+    @g.y_axis_label = "Items"
+
+    stats.each(&@g.method(:data))
+
+    labels = {}
+    stats.days.each_with_index do |d,i|
+      labels[i] = d.to_s
+    end
+    @g.labels = labels
+  end
+
+  def write(path)
+    @g.write(path)
+  end
+end
+
+class Harvester::Chart::StatsPerCollection
+  attr_reader :days
+
+  def initialize
+    @collections = {}
+    @days = []
+  end
+
+  def add_one(collection, day)
+    @days << day unless @days.index(day)
+    collection ||= '(unknown)' # TODO research
+
+    c = @collections[collection] || {}
+    c[day] = (c[day] || 0) + 1
+    @collections[collection] = c
+  end
+
+  def each
+    @collections.each { |n,c|
+      v = []
+      @days.each { |d|
+        v << c[d].to_i
+      }
+
+      yield n, v
+    }
+  end
+end

data/lib/harvester/db.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+
+class Harvester
+  module DB; end
+
+  # creates required database structure
+  def create!
+    task "create database tables" do
+      begin @dbi.transaction do
+        sql_queries(:create).each{ |sql|
+          info "* execute " + File.basename(sql)
+          @dbi.execute File.read(sql)
+        }
+      end; end
+    end
+  end
+
+  # check for feed source changes
+  def maintenance!
+    task "look for sources to purge" do
+      purge = []
+      @dbi.execute("SELECT collection, rss FROM sources").each{ |dbc,dbr|
+        purge << [dbc, dbr] unless (@collections[dbc] || []).include? dbr
+      }
+
+      purge_rss = []
+      purge.each { |c,r|
+        info "* remove #{c}:#{r}..."
+        @dbi.execute "DELETE FROM sources WHERE collection=? AND rss=?", c, r
+        purge_rss << r
+      }
+
+      purge_rss.delete_if { |r|
+        purge_this = true
+
+        @collections.each { |cfc,cfr|
+          if purge_this
+            warn "* must keep #{r} because it's still in #{cfc}" if cfr && cfr.include?(r)
+            purge_this = !(cfr && cfr.include?(r))
+          end
+        }
+
+        !purge_this
+      }
+      purge_rss.each { |r|
+        info "* purge items from feed #{r}"
+        @dbi.execute "DELETE FROM items WHERE rss=?", r
+      }
+    end
+  end
+
+  private
+
+  def update(rss_url, new_source, collection, response, rss_url_nice = rss_url)
+    rss = MRSS.parse(response)
+
+    begin @dbi.transaction do
+      # update source
+      if new_source
+        @dbi.execute "INSERT INTO sources (collection, rss, last, title, link, description) VALUES (?, ?, ?, ?, ?, ?)",
+          collection, rss_url, response['Last-Modified'], rss.title, rss.link, rss.description
+        info rss_url_nice + "Added as source"
+      else
+        @dbi.execute "UPDATE sources SET last=?, title=?, link=?, description=? WHERE collection=? AND rss=?",
+          response['Last-Modified'], rss.title, rss.link, rss.description, collection, rss_url
+        debug rss_url_nice + "Source updated"
+      end
+
+      # update items
+      items_new, items_updated = 0, 0
+      rss.items.each { |item|
+        description = item.description
+
+        # Link mangling
+        begin
+          link = URI::join((rss.link.to_s == '') ? uri.to_s : rss.link.to_s, item.link || rss.link).to_s
+        rescue URI::Error
+          link = item.link
+        end
+
+        # Push into database
+        db_title, = *@dbi.execute("SELECT title FROM items WHERE rss=? AND link=?", rss_url, link).fetch
+
+        if db_title.nil? || db_title.empty? # item is new
+          begin
+            @dbi.execute "INSERT INTO items (rss, title, link, date, description) VALUES (?, ?, ?, ?, ?)",
+              rss_url, item.title, link, item.date.to_s, description
+            items_new += 1
+          #rescue DBI::ProgrammingError
+          #  puts description
+          #  puts "#{$!.class}: #{$!}\n#{$!.backtrace.join("\n")}"
+          end
+        else
+          @dbi.execute "UPDATE items SET title=?, description=? WHERE rss=? AND link=?",
+            item.title, description, rss_url, link
+          items_updated += 1
+        end
+
+        # Remove all enclosures
+        @dbi.execute "DELETE FROM enclosures WHERE rss=? AND link=?", rss_url, link
+
+        # Re-add all enclosures
+        item.enclosures.each do |enclosure|
+          href = URI::join((rss.link.to_s == '') ? link.to_s : rss.link.to_s, enclosure['href']).to_s
+          @dbi.execute "INSERT INTO enclosures (rss, link, href, mime, title, length) VALUES (?, ?, ?, ?, ?, ?)",
+            rss_url, link, href, enclosure['type'], enclosure['title'],
+            !enclosure['length'] || enclosure['length'].empty? ? 0 : enclosure['length']
+        end
+      }
+      info rss_url_nice + "#{ items_new } new items, #{ items_updated } updated"
+    end; end
+  end
+
+  def sql_queries(task)
+    Dir[ File.dirname(__FILE__) + "/../../data/sql/#{ @config['db']['driver'].downcase }/#{ task }*.sql" ].each
+  end
+
+  def sql_query(task)
+    File.dirname(__FILE__) + "/../../data/sql/#{ @config['db']['driver'].downcase }/#{ task }.sql"
+  end
+end

data/lib/harvester/fetch.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+
+require_relative '../harvester'
+require_relative 'db'
+require_relative 'mrss'
+
+require 'eventmachine'
+require 'em-http'
+require 'uri'
+
+class Harvester
+  module FETCH; end
+
+  # fetches new feed updates and store them in the database
+  def fetch!
+    info "FETCH"
+    maintenance! unless @settings['no-maintenance']
+    Fetcher.run @dbi, @collections, @settings, @logger do |*args| update(*args) end # results will be passed to the update function
+  end
+end
+
+# fetches new feed updates and store them in the database using Eventmachine
+module Harvester::Fetcher
+  def self.run(dbi, collections, settings, logger)
+    logger.info '[start] fetch using Eventmachine'
+
+    # prepare logger
+    max_url_size = collections.inject(0){ |acc, (_,rss_urls)| # log display hack
+      if rss_urls
+        max = rss_urls.max_by(&:size).size.to_i
+        acc > max ? acc : max
+      else
+        acc
+      end
+    }
+
+    #dbi['AutoCommit'] = false # TODO check for rdbi
+    
+    EventMachine.run do
+      pending = []
+      collections.each{ |collection, rss_urls|
+        rss_urls and rss_urls.each{ |rss_url|
+          # prepare log prefix
+          rss_url_nice = '* ' + rss_url.ljust(max_url_size) + ' | '
+
+          # get last_modified or if new
+          db_rss, last = dbi.execute("SELECT rss, last FROM sources WHERE collection=? AND rss=?",
+                          collection, rss_url).fetch
+          new_source = db_rss.nil? || db_rss.empty?
+          uri = URI.parse rss_url
+
+          # prepare request
+          header = {}
+          header['Authorization'] = [uri.user, uri.password] if uri.user
+
+          if new_source || last.nil?
+            logger.info rss_url_nice + "GET"
+          else
+            logger.info rss_url_nice + "GET with If-Modified-Since: #{ last }"
+            header['If-Modified-Since'] = last
+          end
+
+          # do request
+          pending << rss_url
+          http = EM::HttpRequest.new(uri).get :head => header
+
+          http.errback do
+            logger.error rss_url_nice + "Request Error: #{ http.error }"
+
+            pending.delete rss_url
+            EM.stop if pending.empty?
+          end
+
+          http.callback do
+            if http.response_header.status != 200
+              logger.warn rss_url_nice + "HTTP not OK, but: #{ http.response_header.status }"
+            elsif http.response.size > settings['size limit'].to_i
+              logger.warn rss_url_nice + "Got too big repsonse: #{ response.size } bytes"
+            else
+              yield rss_url, new_source, collection, http.response, rss_url_nice
+            end
+
+            pending.delete rss_url # same url twice?
+            EM.stop if pending.empty?
+          end
+        }
+      }
+
+      EM.add_timer(settings['timeout'].to_i){
+        pending.each { |rss_url| logger.warn rss_url_nice + 'Timed out' }
+        EM.stop
+      }
+    end
+    logger.info '[done ] fetch using Eventmachine'
+  end
+end