harvester 0.8.0.pre.1

data/CHANGELOG.rdoc

@@ -0,0 +1,45 @@
+= Changelog
+
+== 0.8
+* ...
+
+== 0.7
+* XSLT templates
+* Reworked MRSS
+* HTML entity replacement
+* Transactional database handling
+* Favicon, style, photos
+* Microformats in HTML template
+* Chart timeline
+* Jabberbot fixes
+* HTTPS support optional
+
+== 0.6
+* Complete rewrite (excl. MRSS)
+* YAML configuration
+* Database backend (PostgreSQL)
+* Fetcher is not using wget anymore and supports SSL, HTTP authentication and If-Modified-Since
+* Fetcher operates threaded to request feeds parallelized
+* Minor template variables renaming
+
+== 0.5
+* Fixed a syntax error ;-)
+* Multiple further fixes
+* The Jabber-bot
+* ATOM support by Neingeist
+
+== 0.4
+* Blog variables available in <tmpl:iter-item/>
+* index.html Template readability improvements, CSS now in extra
+  file
+
+== 0.3.2
+* All template elements are now escaped by default. Use
+  escape="false" if you really don't want to escape HTML-Tags and
+  such...
+== 0.3.1
+* Fixed bug where incorrect encoded feeds won't be displayed when
+  parsed for the first time
+
+== 0.3
+* Major overhaul

data/README.rdoc

@@ -0,0 +1,74 @@
+= Harvester 0.8
+The Harvester is a web-based Feed-Aggregator
+
+A running instance can be seen at:
+http://astroblog.spaceboyz.net/harvester/
+
+The Harvester eats the feeds you want and produces a static html/feed page that aggregates all those.
+
+== Installation
+
+Install the harvester with
+
+  gem install harvester
+
+You can now create a new harvester project with
+
+  harvester new <project_directory>
+
+Then you will need to configure <tt>config.yaml</tt>, <tt>collections.yaml</tt> and the templates to suit your needs (see below). You also have to run <tt>harvester db create</tt> (in that directory) to create your initial database structure.
+
+== Configuration
+
+<tt>config.yaml</tt> contains general configuration, like paths, logs and timeouts. You can also configure your database there. The default database backend is SQLite3, but all the following ones are supported:
+* SQLite3
+* PostgreSQL
+* MySQL
+
+The <tt>collections.yaml</tt> file contains the links to your desired feeds.
+
+== Todo
+* Still some things broken after update
+* Improve/fix feed parsing
+* Tidy up templates
+* Security issues
+* Optimize performance
+* Fix jabberbot
+* Implement some kind of tag filters
+
+== Usage
+
+Then run
+
+  harvester run
+
+in the project directory or run
+
+  harvester run -c path/to/config.yaml
+
+This will create the output files. That's it ;).
+
+Usually, you want to run this command automatically every x seconds/minutes. You can use a program like <tt>cron</tt> or simply run
+
+  harvester clock
+
+to start a simple scheduler.
+
+There some more harvester commands to explore. Run
+
+  harvester --help
+
+to get a command list.
+
+== Customizing the templates
+
+...
+
+== Credits
+
+* {Astro}[https://github.com/astro] (2005-2008)
+* Neingeist
+* Tigion
+* Josef Spillner
+* {J-_-L}[https://github.com/janlelis]
+* Others (MySQL patch)

data/Rakefile

@@ -0,0 +1,28 @@
+require 'fileutils'
+GEMSPEC = 'harvester.gemspec'
+
+def gemspec
+  @gemspec ||= eval(File.read( GEMSPEC ), binding, GEMSPEC)
+end
+
+desc "Build the gem"
+task :gem => :gemspec do
+  sh "gem build #{GEMSPEC}"
+  FileUtils.mkdir_p 'pkg'
+  FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", 'pkg'
+end
+
+desc "Install the gem locally (without docs)"
+task :install => :gem do
+  sh %{gem install pkg/#{gemspec.name}-#{gemspec.version} --no-rdoc --no-ri}
+end
+
+desc "Generate the gemspec"
+task :generate do
+  puts gemspec.to_ruby
+end
+
+desc "Validate the gemspec"
+task :gemspec do
+  gemspec.validate
+end

data/bin/harvester

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# call sub-command executable
+if ARGV[0] !~ /^-/ && sub_command = ARGV.shift
+  exec "#{ File.dirname(__FILE__) }/harvester-#{ sub_command }", *ARGV
+end
+
+ARGV << '--help' # if nothing else is set...
+
+# check for options like --version or --help
+require_relative '../lib/harvester'
+Harvester.new_from_argv

data/bin/harvester-chart

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/chart'
+Harvester.new_from_argv.chart!

data/bin/harvester-clock

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester'
+# see https://github.com/adamwiggins/clockwork/
+#     https://github.com/adamwiggins/clockwork/blob/master/bin/clockwork
+require 'clockwork'
+include Clockwork
+
+# TODO monkey patch logger instead of puts
+
+# # # # # 
+
+STDERR.sync = STDOUT.sync = true
+
+harve = Harvester.new_from_argv
+harve.logger.info 'CLOCK'
+
+# get interval
+if ARGV.first
+  interval = ARGV.shift.to_i
+else
+  interval =  harve.settings['interval'] || 2.minutes
+end
+
+harve.logger.info "[start] invoke harvester-run every #{ interval } seconds"
+
+handler do |job|
+  # harve.logger.info '[start] invoke harvester'
+  spawn File.dirname(__FILE__) + '/harvester-run'
+end
+
+every interval, 'run'
+
+run

data/bin/harvester-db

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/db'
+harve = Harvester.new_from_argv
+
+case action = ARGV.shift
+when 'create'
+  harve.create!
+when 'maintenance'
+  harve.maintenance!
+else
+  puts '[DB] Do nothing...'
+end
+  

data/bin/harvester-fetch

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/fetch'
+Harvester.new_from_argv.fetch!

data/bin/harvester-generate

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/generate'
+Harvester.new_from_argv.generate!

data/bin/harvester-jabber

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# TODO: pass differnet username as ARGV
+require_relative '../lib/harvester/jabber'
+Harvester.new_from_argv.jabber!

data/bin/harvester-new

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'fileutils'
+require 'yaml'
+
+if ARGV.empty?
+  puts "USAGE: harvester new <project_dir>"  
+  exit
+end
+
+new    = File.expand_path ARGV[0]
+source = File.expand_path File.dirname(__FILE__) + '/..'
+
+FileUtils.mkdir_p new
+FileUtils.cp_r  source + '/data/templates', new + '/templates'
+FileUtils.cp    source + '/collections.yaml', new + '/collections.yaml'
+FileUtils.cp    source + '/README.rdoc', new + '/README'
+
+# edit default options
+c = YAML::load_file source + '/config.yaml'
+c['settings']['templates'] = 'templates'
+File.open(new + '/config.yaml', 'w'){|out| YAML.dump(c, out) }
+
+puts 'Created new harvester project at ' + ARGV[0]

data/bin/harvester-post

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require_relative '../lib/harvester/post'
+Harvester.new_from_argv.post! ARGV.shift

data/bin/harvester-run

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# run a complete harvester update :)
+require_relative '../lib/harvester/fetch'
+require_relative '../lib/harvester/generate'
+require_relative '../lib/harvester/chart'
+require_relative '../lib/harvester/post'
+
+harve = Harvester.new_from_argv
+harve.fetch!
+harve.generate!
+harve.chart! unless harve.settings['no-chart']
+harve.post!

data/collections.yaml

@@ -0,0 +1,15 @@
+---
+# this file contains all feed urls 
+blogs: # main feeds
+#-
+ticker: # the ticker only shows headlines
+#-
+projects: # e.g. github
+#- http://github.com/feeds/astro/commits/harvester/blog-harvester.de
+links: # e.g. delicious
+#- http://feeds.delicious.com/rss/YYYYYYYYYYYYYY
+photos: # e.g. flickr
+#- "http://api.flickr.com/services/feeds/photos_public.gne?id=XXXXXXXX@N00&format=atom"
+microblogs: # twitter & identi.ca
+#- http://twitter.com/statuses/user_timeline/XXXXXXXX.rss
+#- http://identi.ca/api/statuses/user_timeline/YYYYYYYYYYY.atom

data/config.yaml

@@ -0,0 +1,13 @@
+---
+settings:
+  collections: collections.yaml
+  templates: data/templates
+  output: html
+  size limit: 202400
+  interval: 100
+  timeout: 20
+  log_file: STDOUT
+  log_level: debug
+db:
+  driver: SQLite3
+  database: harvester.sqlite3

data/data/ent/HTMLlat1.ent

@@ -0,0 +1,194 @@
+<!-- Portions (C) International Organization for Standardization 1986
+     Permission to copy in any form is granted for use with
+     conforming SGML systems and applications as defined in
+     ISO 8879, provided this notice is included in all copies.
+-->
+<!-- Character entity set. Typical invocation:
+     <!ENTITY % HTMLlat1 PUBLIC
+       "-//W3C//ENTITIES Latin 1//EN//HTML">
+     %HTMLlat1;
+-->
+
+<!ENTITY nbsp   CDATA "&#160;" -- no-break space = non-breaking space,
+                                  U+00A0 ISOnum -->
+<!ENTITY iexcl  CDATA "&#161;" -- inverted exclamation mark, U+00A1 ISOnum -->
+<!ENTITY cent   CDATA "&#162;" -- cent sign, U+00A2 ISOnum -->
+<!ENTITY pound  CDATA "&#163;" -- pound sign, U+00A3 ISOnum -->
+<!ENTITY curren CDATA "&#164;" -- currency sign, U+00A4 ISOnum -->
+<!ENTITY yen    CDATA "&#165;" -- yen sign = yuan sign, U+00A5 ISOnum -->
+<!ENTITY brvbar CDATA "&#166;" -- broken bar = broken vertical bar,
+                                  U+00A6 ISOnum -->
+<!ENTITY sect   CDATA "&#167;" -- section sign, U+00A7 ISOnum -->
+<!ENTITY uml    CDATA "&#168;" -- diaeresis = spacing diaeresis,
+                                  U+00A8 ISOdia -->
+<!ENTITY copy   CDATA "&#169;" -- copyright sign, U+00A9 ISOnum -->
+<!ENTITY ordf   CDATA "&#170;" -- feminine ordinal indicator, U+00AA ISOnum -->
+<!ENTITY laquo  CDATA "&#171;" -- left-pointing double angle quotation mark
+                                  = left pointing guillemet, U+00AB ISOnum -->
+<!ENTITY not    CDATA "&#172;" -- not sign, U+00AC ISOnum -->
+<!ENTITY shy    CDATA "&#173;" -- soft hyphen = discretionary hyphen,
+                                  U+00AD ISOnum -->
+<!ENTITY reg    CDATA "&#174;" -- registered sign = registered trade mark sign,
+                                  U+00AE ISOnum -->
+<!ENTITY macr   CDATA "&#175;" -- macron = spacing macron = overline
+                                  = APL overbar, U+00AF ISOdia -->
+<!ENTITY deg    CDATA "&#176;" -- degree sign, U+00B0 ISOnum -->
+<!ENTITY plusmn CDATA "&#177;" -- plus-minus sign = plus-or-minus sign,
+                                  U+00B1 ISOnum -->
+<!ENTITY sup2   CDATA "&#178;" -- superscript two = superscript digit two
+                                  = squared, U+00B2 ISOnum -->
+<!ENTITY sup3   CDATA "&#179;" -- superscript three = superscript digit three
+                                  = cubed, U+00B3 ISOnum -->
+<!ENTITY acute  CDATA "&#180;" -- acute accent = spacing acute,
+                                  U+00B4 ISOdia -->
+<!ENTITY micro  CDATA "&#181;" -- micro sign, U+00B5 ISOnum -->
+<!ENTITY para   CDATA "&#182;" -- pilcrow sign = paragraph sign,
+                                  U+00B6 ISOnum -->
+<!ENTITY middot CDATA "&#183;" -- middle dot = Georgian comma
+                                  = Greek middle dot, U+00B7 ISOnum -->
+<!ENTITY cedil  CDATA "&#184;" -- cedilla = spacing cedilla, U+00B8 ISOdia -->
+<!ENTITY sup1   CDATA "&#185;" -- superscript one = superscript digit one,
+                                  U+00B9 ISOnum -->
+<!ENTITY ordm   CDATA "&#186;" -- masculine ordinal indicator,
+                                  U+00BA ISOnum -->
+<!ENTITY raquo  CDATA "&#187;" -- right-pointing double angle quotation mark
+                                  = right pointing guillemet, U+00BB ISOnum -->
+<!ENTITY frac14 CDATA "&#188;" -- vulgar fraction one quarter
+                                  = fraction one quarter, U+00BC ISOnum -->
+<!ENTITY frac12 CDATA "&#189;" -- vulgar fraction one half
+                                  = fraction one half, U+00BD ISOnum -->
+<!ENTITY frac34 CDATA "&#190;" -- vulgar fraction three quarters
+                                  = fraction three quarters, U+00BE ISOnum -->
+<!ENTITY iquest CDATA "&#191;" -- inverted question mark
+                                  = turned question mark, U+00BF ISOnum -->
+<!ENTITY Agrave CDATA "&#192;" -- latin capital letter A with grave
+                                  = latin capital letter A grave,
+                                  U+00C0 ISOlat1 -->
+<!ENTITY Aacute CDATA "&#193;" -- latin capital letter A with acute,
+                                  U+00C1 ISOlat1 -->
+<!ENTITY Acirc  CDATA "&#194;" -- latin capital letter A with circumflex,
+                                  U+00C2 ISOlat1 -->
+<!ENTITY Atilde CDATA "&#195;" -- latin capital letter A with tilde,
+                                  U+00C3 ISOlat1 -->
+<!ENTITY Auml   CDATA "&#196;" -- latin capital letter A with diaeresis,
+                                  U+00C4 ISOlat1 -->
+<!ENTITY Aring  CDATA "&#197;" -- latin capital letter A with ring above
+                                  = latin capital letter A ring,
+                                  U+00C5 ISOlat1 -->
+<!ENTITY AElig  CDATA "&#198;" -- latin capital letter AE
+                                  = latin capital ligature AE,
+                                  U+00C6 ISOlat1 -->
+<!ENTITY Ccedil CDATA "&#199;" -- latin capital letter C with cedilla,
+                                  U+00C7 ISOlat1 -->
+<!ENTITY Egrave CDATA "&#200;" -- latin capital letter E with grave,
+                                  U+00C8 ISOlat1 -->
+<!ENTITY Eacute CDATA "&#201;" -- latin capital letter E with acute,
+                                  U+00C9 ISOlat1 -->
+<!ENTITY Ecirc  CDATA "&#202;" -- latin capital letter E with circumflex,
+                                  U+00CA ISOlat1 -->
+<!ENTITY Euml   CDATA "&#203;" -- latin capital letter E with diaeresis,
+                                  U+00CB ISOlat1 -->
+<!ENTITY Igrave CDATA "&#204;" -- latin capital letter I with grave,
+                                  U+00CC ISOlat1 -->
+<!ENTITY Iacute CDATA "&#205;" -- latin capital letter I with acute,
+                                  U+00CD ISOlat1 -->
+<!ENTITY Icirc  CDATA "&#206;" -- latin capital letter I with circumflex,
+                                  U+00CE ISOlat1 -->
+<!ENTITY Iuml   CDATA "&#207;" -- latin capital letter I with diaeresis,
+                                  U+00CF ISOlat1 -->
+<!ENTITY ETH    CDATA "&#208;" -- latin capital letter ETH, U+00D0 ISOlat1 -->
+<!ENTITY Ntilde CDATA "&#209;" -- latin capital letter N with tilde,
+                                  U+00D1 ISOlat1 -->
+<!ENTITY Ograve CDATA "&#210;" -- latin capital letter O with grave,
+                                  U+00D2 ISOlat1 -->
+<!ENTITY Oacute CDATA "&#211;" -- latin capital letter O with acute,
+                                  U+00D3 ISOlat1 -->
+<!ENTITY Ocirc  CDATA "&#212;" -- latin capital letter O with circumflex,
+                                  U+00D4 ISOlat1 -->
+<!ENTITY Otilde CDATA "&#213;" -- latin capital letter O with tilde,
+                                  U+00D5 ISOlat1 -->
+<!ENTITY Ouml   CDATA "&#214;" -- latin capital letter O with diaeresis,
+                                  U+00D6 ISOlat1 -->
+<!ENTITY times  CDATA "&#215;" -- multiplication sign, U+00D7 ISOnum -->
+<!ENTITY Oslash CDATA "&#216;" -- latin capital letter O with stroke
+                                  = latin capital letter O slash,
+                                  U+00D8 ISOlat1 -->
+<!ENTITY Ugrave CDATA "&#217;" -- latin capital letter U with grave,
+                                  U+00D9 ISOlat1 -->
+<!ENTITY Uacute CDATA "&#218;" -- latin capital letter U with acute,
+                                  U+00DA ISOlat1 -->
+<!ENTITY Ucirc  CDATA "&#219;" -- latin capital letter U with circumflex,
+                                  U+00DB ISOlat1 -->
+<!ENTITY Uuml   CDATA "&#220;" -- latin capital letter U with diaeresis,
+                                  U+00DC ISOlat1 -->
+<!ENTITY Yacute CDATA "&#221;" -- latin capital letter Y with acute,
+                                  U+00DD ISOlat1 -->
+<!ENTITY THORN  CDATA "&#222;" -- latin capital letter THORN,
+                                  U+00DE ISOlat1 -->
+<!ENTITY szlig  CDATA "&#223;" -- latin small letter sharp s = ess-zed,
+                                  U+00DF ISOlat1 -->
+<!ENTITY agrave CDATA "&#224;" -- latin small letter a with grave
+                                  = latin small letter a grave,
+                                  U+00E0 ISOlat1 -->
+<!ENTITY aacute CDATA "&#225;" -- latin small letter a with acute,
+                                  U+00E1 ISOlat1 -->
+<!ENTITY acirc  CDATA "&#226;" -- latin small letter a with circumflex,
+                                  U+00E2 ISOlat1 -->
+<!ENTITY atilde CDATA "&#227;" -- latin small letter a with tilde,
+                                  U+00E3 ISOlat1 -->
+<!ENTITY auml   CDATA "&#228;" -- latin small letter a with diaeresis,
+                                  U+00E4 ISOlat1 -->
+<!ENTITY aring  CDATA "&#229;" -- latin small letter a with ring above
+                                  = latin small letter a ring,
+                                  U+00E5 ISOlat1 -->
+<!ENTITY aelig  CDATA "&#230;" -- latin small letter ae
+                                  = latin small ligature ae, U+00E6 ISOlat1 -->
+<!ENTITY ccedil CDATA "&#231;" -- latin small letter c with cedilla,
+                                  U+00E7 ISOlat1 -->
+<!ENTITY egrave CDATA "&#232;" -- latin small letter e with grave,
+                                  U+00E8 ISOlat1 -->
+<!ENTITY eacute CDATA "&#233;" -- latin small letter e with acute,
+                                  U+00E9 ISOlat1 -->
+<!ENTITY ecirc  CDATA "&#234;" -- latin small letter e with circumflex,
+                                  U+00EA ISOlat1 -->
+<!ENTITY euml   CDATA "&#235;" -- latin small letter e with diaeresis,
+                                  U+00EB ISOlat1 -->
+<!ENTITY igrave CDATA "&#236;" -- latin small letter i with grave,
+                                  U+00EC ISOlat1 -->
+<!ENTITY iacute CDATA "&#237;" -- latin small letter i with acute,
+                                  U+00ED ISOlat1 -->
+<!ENTITY icirc  CDATA "&#238;" -- latin small letter i with circumflex,
+                                  U+00EE ISOlat1 -->
+<!ENTITY iuml   CDATA "&#239;" -- latin small letter i with diaeresis,
+                                  U+00EF ISOlat1 -->
+<!ENTITY eth    CDATA "&#240;" -- latin small letter eth, U+00F0 ISOlat1 -->
+<!ENTITY ntilde CDATA "&#241;" -- latin small letter n with tilde,
+                                  U+00F1 ISOlat1 -->
+<!ENTITY ograve CDATA "&#242;" -- latin small letter o with grave,
+                                  U+00F2 ISOlat1 -->
+<!ENTITY oacute CDATA "&#243;" -- latin small letter o with acute,
+                                  U+00F3 ISOlat1 -->
+<!ENTITY ocirc  CDATA "&#244;" -- latin small letter o with circumflex,
+                                  U+00F4 ISOlat1 -->
+<!ENTITY otilde CDATA "&#245;" -- latin small letter o with tilde,
+                                  U+00F5 ISOlat1 -->
+<!ENTITY ouml   CDATA "&#246;" -- latin small letter o with diaeresis,
+                                  U+00F6 ISOlat1 -->
+<!ENTITY divide CDATA "&#247;" -- division sign, U+00F7 ISOnum -->
+<!ENTITY oslash CDATA "&#248;" -- latin small letter o with stroke,
+                                  = latin small letter o slash,
+                                  U+00F8 ISOlat1 -->
+<!ENTITY ugrave CDATA "&#249;" -- latin small letter u with grave,
+                                  U+00F9 ISOlat1 -->
+<!ENTITY uacute CDATA "&#250;" -- latin small letter u with acute,
+                                  U+00FA ISOlat1 -->
+<!ENTITY ucirc  CDATA "&#251;" -- latin small letter u with circumflex,
+                                  U+00FB ISOlat1 -->
+<!ENTITY uuml   CDATA "&#252;" -- latin small letter u with diaeresis,
+                                  U+00FC ISOlat1 -->
+<!ENTITY yacute CDATA "&#253;" -- latin small letter y with acute,
+                                  U+00FD ISOlat1 -->
+<!ENTITY thorn  CDATA "&#254;" -- latin small letter thorn,
+                                  U+00FE ISOlat1 -->
+<!ENTITY yuml   CDATA "&#255;" -- latin small letter y with diaeresis,
+                                  U+00FF ISOlat1 -->