feed2imap 1.2.6

data/bin/feed2imap

@@ -0,0 +1,49 @@
+#!/usr/bin/ruby
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'feed2imap/feed2imap'
+require 'optparse'
+
+verbose = false
+version = false
+cacherebuild = false
+configf = ENV['HOME'] + '/.feed2imaprc'
+progname = File::basename($PROGRAM_NAME)
+opts = OptionParser::new do |opts|
+  opts.program_name = progname
+  opts.banner = "Usage: #{progname} [options]"
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-v", "--verbose", "Verbose mode") do |v|
+    verbose = true
+  end
+
+  opts.on("-d", "--debug", "Debug mode") do |v|
+    verbose = :debug
+  end
+
+  opts.on("-V", "--version", "Display Feed2Imap version") do |v|
+    version = true
+  end
+  opts.on("-c", "--rebuild-cache", "Cache rebuilding run : will fetch everything and add to cache, without uploading to the IMAP server. Useful if your cache file was lost, and you don't want to re-read all the items.") do |c|
+    cacherebuild = true
+  end
+  opts.on("-f", "--config <file>", "Select alternate config file") do |f|
+    configf = f
+  end
+end
+begin
+  opts.parse!(ARGV)
+rescue OptionParser::ParseError => pe
+  opts.warn pe
+  puts opts
+  exit 1
+end
+
+if version
+  puts "Feed2Imap v.#{Feed2Imap::VERSION}"
+else
+  Feed2Imap::new(verbose, cacherebuild, configf)
+end

data/bin/feed2imap-cleaner

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'feed2imap/feed2imap'
+require 'optparse'
+
+configf = ENV['HOME'] + '/.feed2imaprc'
+dryrun = false
+
+opts = OptionParser::new do |opts|
+  opts.banner = "Usage: feed2imap-cleaner [options]"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-d", "--dry-run", "Dont really remove messages") do |v|
+    dryrun = true
+  end
+  opts.on("-f", "--config <file>", "Select alternate config file") do |f|
+    configf = f
+  end
+end
+opts.parse!(ARGV)
+
+config = nil
+File::open(configf) { |f| config = F2IConfig::new(f) }
+config.imap_accounts.each_value do |ac|
+  ac.connect
+end
+config.feeds.each do |f|
+  f.imapaccount.cleanup(f.folder, dryrun)
+end
+

data/bin/feed2imap-dumpconfig

@@ -0,0 +1,42 @@
+#!/usr/bin/ruby
+
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'feed2imap/config'
+require 'optparse'
+
+configf = ENV['HOME'] + '/.feed2imaprc'
+opts = OptionParser::new do |opts|
+  opts.banner = "Usage: ./dumpconfig.rb [options]"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-f", "--config <file>", "Select alternate config file") do |f|
+    configf = f
+  end
+end
+opts.parse!(ARGV)
+
+if not File::exist?(configf)
+  puts "Configuration file #{configfile} not found."
+  exit(1)
+end
+File::open(configf) { |f|  puts F2IConfig::new(f).to_s }

data/bin/feed2imap-opmlimport

@@ -0,0 +1,48 @@
+#!/usr/bin/ruby
+
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rexml/document'
+require 'yaml'
+
+DEFAULTIMAPFOLDER = 'imap://login:password@imapserver/folder.folder2'
+
+opml = ARGV[0]
+doc = nil
+doc = REXML::Document::new(IO.read(opml))
+feeds = []
+doc.root.each_element('//outline') do |e|
+  if u = e.attribute('xmlUrl') || e.attribute('htmlUrl')
+    # dirty liferea hack
+    next if u.value == 'vfolder'
+    # get title
+    t = e.attribute('text') || e.attribute('Title') || nil
+    if t.nil?
+      title = '*** FEED TITLE (must be unique) ***'
+    else
+      title = t.value
+    end
+    url = u.value
+    feeds.push({'name' => title, 'url' => url, 'target' => DEFAULTIMAPFOLDER})
+  end
+end
+YAML::dump({'feeds' => feeds}, $stdout)

data/data/doc/feed2imap/examples/feed2imaprc

@@ -0,0 +1,72 @@
+# Global options:
+# max-failures: maximum number of failures allowed before they are reported in
+#   normal mode (default 10). By default, failures are only visible in verbose
+#   mode. Most feeds tend to suffer from temporary failures.
+# dumpdir: (for debugging purposes) directory where all fetched feeds will be
+#   dumped.
+# debug-updated: (for debugging purposes) if true, display a lot of information
+#   about the "updated-items" algorithm.
+# include-images: download images and include them in the mail? (true/false)
+# reupload-if-updated: when an item is updated, and was previously deleted,
+#   reupload it? (true/false, default true)
+# default-email: default email address in the format foo@example.com
+# disable-ssl-verification: disable SSL certification when connecting
+#   to IMAPS accounts (true/false)
+# timeout: time before getting timeout when fetching feeds (default 30) in seconds
+# 
+# Per-feed options:
+# name: name of the feed (must be unique)
+# url: HTTP[S] address where the feed has to be fetched
+# target: the IMAP URI where to put emails. Should start with imap:// for IMAP,
+#   imaps:// for IMAPS and maildir:// for a path to a local maildir.
+# min-frequency: (in HOURS) is the minimum frequency with which this particular
+#   feed will be fetched
+# disable: if set to something, the feed will be ignored
+# include-images: download images and include them in the mail? (true/false)
+# reupload-if-updated: when an item is updated, and was previously deleted,
+#   reupload it? (true/false, default true)
+# always-new: feed2imap tries to use a clever algorithm to determine whether
+#   an item is new or has been updated. It doesn't work well with some web apps
+#   like mediawiki. When this flag is enabled, all items which don't match
+#   exactly a previously downloaded item are considered as new items.
+# ignore-hash: Some feeds change the content of their items all the time, so
+#   feed2imap detects that they have been updated at each run. When this flag
+#   is enabled, feed2imap ignores the content of an item when determining
+#   whether the item is already known.
+# dumpdir: (for debugging purposes) directory where all fetched feeds will be
+#   dumped.
+# Snownews/Liferea scripts support :
+#   execurl: Command to execute that will display the RSS/Atom feed on stdout
+#   filter: Command to execute which will receive the RSS/Atom feed on stdin,
+#    modify it, and output it on stdout.
+#   For more information: http://kiza.kcore.de/software/snownews/snowscripts/
+#
+#
+# If your login contains an @ character, replace it with %40. Other reserved
+# characters can be escaped in the same way (see man ascii to get their code)
+feeds: 
+  - name: feed2imap
+    url: http://home.gna.org/feed2imap/feed2imap.rss
+    target: imap://luser:password@imap.apinc.org/INBOX.Feeds.Feed2Imap
+  - name: lucas
+    url: http://www.lucas-nussbaum.net/blog/?feed=rss2
+    target: imap://luser:password@imap.apinc.org/INBOX.Feeds.Lucas
+  - name: JabberFrWiki
+    url: http://wiki.jabberfr.org/index.php?title=Special:Recentchanges&feed=rss
+    target: imaps://luser:password@imap.apinc.org/INBOX.Feeds.JabberFR
+    always-new: true
+  - name: LeMonde
+    execurl: "wget -q -O /dev/stdout http://www.lemonde.fr/rss/sequence/0,2-3208,1-0,0.xml"
+    filter: "/home/lucas/lemonde_getbody"
+    target: imap://luser:password@imap.apinc.org/INBOX.Feeds.LeMonde
+# It is also possible to reuse the same string in the target parameter:
+# target-refix: &target "imap://user:pass@host/rss."
+# feeds:
+#   - name: test1
+#     target: [ *target, 'test1' ]
+#     ...
+#   - name: test2
+#     target: [ *target, 'test2' ]
+#     ...
+
+# vim: ft=yaml:sts=2:expandtab

data/data/man/man1/feed2imap-cleaner.1

@@ -0,0 +1,43 @@
+.TH feed2imap\-cleaner 1 "Jul 25, 2005"  
+.SH NAME
+feed2imap\-cleaner \- Removes old items from IMAP folders
+.SH SYNOPSIS
+\fBfeed2imap\-cleaner\fR [OPTIONS]
+.SH DESCRIPTION
+feed2imap\-cleaner deletes old items from IMAP folders specified in the configuration file. The actual query string used to determine whether an item is old is :
+"SEEN NOT FLAGGED BEFORE (3 days ago)". Which means that an item WON'T be deleted if it satisfies one of the following conditions :
+.TP 0.2i
+\(bu
+It isn't 3 days old ;
+.TP 0.2i
+\(bu
+It hasn't been read yet ;
+.TP 0.2i
+\(bu
+It is flagged (marked as Important, for example).
+.TP 
+\fB\-d\fR, \fB\-\-dry\-run\fR 
+Don't remove anything, but show what would be removed if run without this option.
+.TP 
+\fB\-f\fR, \fB\-\-config \fIfile\fB\fR 
+Use another config file (~/.feed2imaprc is the default).
+.SH BUGS
+Deletion criterias should probably be more configurable.
+.SH "SEE ALSO"
+Homepage : 
+http://home.gna.org/feed2imap/
+.PP
+\fBfeed2imaprc\fR(5),
+\fBfeed2imap\fR(1)
+.SH AUTHOR
+Copyright (C) 2005 Lucas Nussbaum lucas@lucas\-nussbaum.net
+.PP
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.

data/data/man/man1/feed2imap-dumpconfig.1

@@ -0,0 +1,28 @@
+.TH feed2imap\-dumpconfig 1 "Jul 25, 2005"  
+.SH NAME
+feed2imap\-dumpconfig \- Dump feed2imap config
+.SH SYNOPSIS
+\fBfeed2imap\-dumpconfig\fR [OPTIONS]
+.SH DESCRIPTION
+feed2imap\-dumpconfig dumps the content of your feed2imaprc to screen.
+.TP 
+\fB\-f\fR, \fB\-\-config \fIfile\fB\fR 
+Use another config file (~/.feed2imaprc is the default).
+.SH "SEE ALSO"
+Homepage : 
+http://home.gna.org/feed2imap/
+.PP
+\fBfeed2imaprc\fR(5),
+\fBfeed2imap\fR(1)
+.SH AUTHOR
+Copyright (C) 2005 Lucas Nussbaum lucas@lucas\-nussbaum.net
+.PP
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.

data/data/man/man1/feed2imap-opmlimport.1

@@ -0,0 +1,27 @@
+.TH feed2imap\-opmlimport 1 "Jul 25, 2005"  
+.SH NAME
+feed2imap\-opmlimport \- Convert an OPML subscription list to a feed2imap config file
+.SH SYNOPSIS
+\fBfeed2imap\-opmlimport\fR
+.SH DESCRIPTION
+feed2imap\-opmlimport reads an OPML subscription list on standard input and outputs a feed2imap configuration file on standard output. The resulting configuration file will require some tweaking.
+.SH BUGS
+Should probably accept parameters to be able to change default values.
+.SH "SEE ALSO"
+Homepage : 
+http://home.gna.org/feed2imap/
+.PP
+\fBfeed2imaprc\fR(5),
+\fBfeed2imap\fR(1)
+.SH AUTHOR
+Copyright (C) 2005 Lucas Nussbaum lucas@lucas\-nussbaum.net
+.PP
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.

data/data/man/man1/feed2imap.1

@@ -0,0 +1,42 @@
+.TH feed2imap 1 "Jul 25, 2005"  
+.SH NAME
+feed2imap \- clever RSS/ATOM feed aggregator
+.SH SYNOPSIS
+\fBfeed2imap\fR [OPTIONS]
+.SH DESCRIPTION
+feed2imap is an RSS/Atom feed aggregator. After
+Downloading feeds (over HTTP or HTTPS), it uploads them to a specified
+folder of an IMAP mail server. The user can then access the feeds using
+Mutt, Evolution, Mozilla Thunderbird or even a webmail.
+.TP 
+\fB\-V\fR, \fB\-\-version\fR 
+Show version information.
+.TP 
+\fB\-v\fR, \fB\-\-verbose\fR 
+Run in verbose mode.
+.TP 
+\fB\-c\fR, \fB\-\-rebuild\-cache\fR 
+Rebuilds the cache. Fetches all items and mark them as already seen. Useful if you lose your .feed2imap.cache file.
+.TP 
+\fB\-f\fR, \fB\-\-config \fIfile\fB\fR 
+Use another config file (~/.feed2imaprc is the default).
+.SH "SEE ALSO"
+Homepage : 
+http://home.gna.org/feed2imap/
+.PP
+\fBfeed2imaprc\fR(5),
+\fBfeed2imap\-cleaner\fR(1),
+\fBfeed2imap\-dumpconfig\fR(1),
+\fBfeed2imap\-opmlimport\fR(1)
+.SH AUTHOR
+Copyright (C) 2005 Lucas Nussbaum lucas@lucas\-nussbaum.net
+.PP
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.

data/data/man/man5/feed2imaprc.5

@@ -0,0 +1,29 @@
+.TH feed2imaprc 5 "Jul 25, 2005"  
+.SH NAME
+feed2imaprc \- feed2imap configuration file
+.SH SYNOPSIS
+\fBfeed2imaprc\fR is feed2imap's configuration file. It is usually located in \fB~/.feed2imaprc\fR.
+.SH EXAMPLE
+See \fB/usr/share/doc/feed2imap/examples/feed2imaprc\fR.
+.SH "RESERVED CHARACTERS"
+Some characters are reserved in RFC2396 (URI). If you need to include a reserved character in the login/password part of your target URI, replace it with its hex code. For example, @ can be replaced by %40.
+.SH BUGS
+This manpage should probably give more details. However, the example configuration file is
+very well documented.
+.SH "SEE ALSO"
+Homepage : 
+http://home.gna.org/feed2imap/
+.PP
+\fBfeed2imap\fR(1)
+.SH AUTHOR
+Copyright (C) 2005 Lucas Nussbaum lucas@lucas\-nussbaum.net
+.PP
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+.PP
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.

data/lib/feed2imap.rb

@@ -0,0 +1 @@
+require 'feed2imap/feed2imap'

data/lib/feed2imap/cache.rb

@@ -0,0 +1,302 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+# debug mode
+$updateddebug = false
+
+# This class manages a cache of items
+# (items which have already been seen)
+
+require 'digest/md5'
+
+class ItemCache
+  def initialize(debug = false)
+    @channels = {}
+    @@cacheidx = 0
+    $updateddebug = debug
+    self
+  end
+
+  # Returns the really new items amongst items
+  def get_new_items(id, items, always_new = false, ignore_hash = false)
+    if $updateddebug
+      puts "======================================================="
+      puts "GET_NEW_ITEMS FOR #{id}... (#{Time::now})"
+    end
+    @channels[id] ||= CachedChannel::new
+    @channels[id].parsefailures = 0
+    return @channels[id].get_new_items(items, always_new, ignore_hash)
+  end
+
+  # Commit changes to the cache
+  def commit_cache(id)
+    @channels[id] ||= CachedChannel::new
+    @channels[id].commit
+  end
+
+  # Get the last time the cache was updated
+  def get_last_check(id)
+    @channels[id] ||= CachedChannel::new
+    @channels[id].lastcheck
+  end
+
+  # Get the last time the cache was updated
+  def set_last_check(id, time)
+    @channels[id] ||= CachedChannel::new
+    @channels[id].lastcheck = time
+    @channels[id].failures = 0
+    self
+  end
+
+  # Fetching failure.
+  # returns number of failures
+  def fetch_failed(id)
+    @channels[id].fetch_failed
+  end
+
+  # Parsing failure.
+  # returns number of failures
+  def parse_failed(id)
+    @channels[id].parse_failed
+  end
+
+  # Load the cache from an IO stream
+  def load(io)
+    begin
+      @@cacheidx, @channels = Marshal.load(io)
+    rescue
+      @channels = Marshal.load(io)
+      @@cacheidx = 0
+    end
+  end
+
+  # Save the cache to an IO stream
+  def save(io)
+    Marshal.dump([@@cacheidx, @channels], io)
+  end
+  
+  # Return the number of channels in the cache
+  def nbchannels
+    @channels.length
+  end
+
+  # Return the number of items in the cache
+  def nbitems
+    nb = 0
+    @channels.each_value { |c|
+      nb += c.nbitems
+    }
+    nb
+  end
+
+  def ItemCache.getindex
+    i = @@cacheidx
+    @@cacheidx += 1
+    i
+  end
+end
+
+class CachedChannel
+  # Size of the cache for each feed
+  # 100 items should be enough for everybody, even quite busy feeds
+  CACHESIZE = 100
+
+  attr_accessor :lastcheck, :items, :failures, :parsefailures
+
+  def initialize
+    @lastcheck = Time::at(0)
+    @items = []
+    @itemstemp = [] # see below
+    @nbnewitems = 0
+    @failures = 0
+    @parsefailures = 0
+  end
+
+  # Let's explain @items and @itemstemp.
+  # @items contains the CachedItems serialized to the disk cache.
+  # The - quite complicated - get_new_items method fills in @itemstemp
+  # but leaves @items unchanged.
+  # Later, the commit() method replaces @items with @itemstemp and
+  # empties @itemstemp. This way, if something wrong happens during the
+  # upload to the IMAP server, items aren't lost.
+  # @nbnewitems is set by get_new_items, and is used to limit the number
+  # of (old) items serialized.
+
+  # Returns the really new items amongst items
+  def get_new_items(items, always_new = false, ignore_hash = false)
+    # save number of new items
+    @nbnewitems = items.length
+    # set items' cached version if not set yet
+    newitems = []
+    updateditems = []
+    @itemstemp = @items
+    items.each { |i| i.cacheditem ||= CachedItem::new(i) }
+    if $updateddebug
+      puts "-------Items downloaded before dups removal (#{items.length}) :----------"
+      items.each { |i| puts "#{i.cacheditem.to_s}" }
+    end
+    # remove dups
+    dups = true
+    while dups
+      dups = false
+      for i in 0...items.length do
+        for j in i+1...items.length do
+          if items[i].cacheditem == items[j].cacheditem
+            if $updateddebug
+              puts "## Removed duplicate #{items[j].cacheditem.to_s}"
+            end
+            items.delete_at(j)
+            dups = true
+            break
+          end
+        end
+        break if dups
+      end
+    end
+    # debug : dump interesting info to stdout.
+    if $updateddebug
+      puts "-------Items downloaded after dups removal (#{items.length}) :----------"
+      items.each { |i| puts "#{i.cacheditem.to_s}" }
+      puts "-------Items already there (#{@items.length}) :----------"
+      @items.each { |i| puts "#{i.to_s}" }
+      puts "Items always considered as new: #{always_new.to_s}"
+      puts "Items compared ignoring the hash: #{ignore_hash.to_s}"
+    end
+    items.each do |i|
+      found = false
+      # Try to find a perfect match
+      @items.each do |j|
+        # note that simple_compare only CachedItem, not RSSItem, so we have to use
+        # j.simple_compare(i) and not i.simple_compare(j)
+        if (i.cacheditem == j and not ignore_hash) or
+            (j.simple_compare(i) and ignore_hash)
+          i.cacheditem.index = j.index
+          found = true
+          # let's put j in front of itemstemp
+          @itemstemp.delete(j)
+          @itemstemp.unshift(j)
+          break
+        end
+        # If we didn't find exact match, try to check if we have an update
+        if j.is_ancestor_of(i)
+          i.cacheditem.index = j.index
+          i.cacheditem.updated = true
+          updateditems.push(i)
+          found = true
+          # let's put j in front of itemstemp
+          @itemstemp.delete(j)
+          @itemstemp.unshift(i.cacheditem)
+          break
+        end
+      end
+      next if found
+      # add as new
+      i.cacheditem.create_index
+      newitems.push(i)
+      # add i.cacheditem to @itemstemp
+      @itemstemp.unshift(i.cacheditem)
+    end
+    if $updateddebug
+      puts "-------New items :----------"
+      newitems.each { |i| puts "#{i.cacheditem.to_s}" }
+      puts "-------Updated items :----------"
+      updateditems.each { |i| puts "#{i.cacheditem.to_s}" }
+    end
+    return [newitems, updateditems]
+  end
+  
+  def commit
+    # too old items must be dropped
+    n = @nbnewitems > CACHESIZE ? @nbnewitems : CACHESIZE
+    @items = @itemstemp[0..n]
+    if $updateddebug
+      puts "Committing: new items: #{@nbnewitems} / items kept: #{@items.length}"
+    end
+    @itemstemp = []
+    self
+  end
+
+  # returns the number of items
+  def nbitems
+    @items.length
+  end
+
+  def parse_failed
+    @parsefailures = 0 if @parsefailures.nil?
+    @parsefailures += 1
+    return @parsefailures
+  end
+
+  def fetch_failed
+    @failures = 0 if @failures.nil?
+    @failures += 1
+    return @failures
+  end
+end
+
+# This class is the only thing kept in the cache
+class CachedItem
+  attr_reader :title, :link, :creator, :date, :hash
+  attr_accessor :index
+  attr_accessor :updated
+
+  def initialize(item)
+    @title = item.title
+    @link = item.link
+    @date = item.date
+    @creator = item.creator
+    if item.content.nil?
+      @hash = nil
+    else
+      @hash = Digest::MD5.hexdigest(item.content.to_s)
+    end
+  end
+
+  def ==(other)
+    if $updateddebug
+      puts "Comparing #{self.to_s} and #{other.to_s}:"
+      puts "Title: #{@title == other.title}"
+      puts "Link: #{@link == other.link}"
+      puts "Creator: #{@creator == other.creator}"
+      puts "Date: #{@date == other.date}"
+      puts "Hash: #{@hash == other.hash}"
+    end
+    @title == other.title and @link == other.link and
+        (@creator.nil? or other.creator.nil? or @creator == other.creator) and
+	(@date.nil? or other.date.nil? or @date == other.date) and @hash == other.hash
+  end
+
+  def simple_compare(other)
+    @title == other.title and @link == other.link and
+        (@creator.nil? or other.creator.nil? or @creator == other.creator) 
+  end
+
+  def create_index
+    @index = ItemCache.getindex
+  end
+
+  def is_ancestor_of(other)
+    (@link and other.link and @link == other.link) and
+      ((@creator and other.creator and @creator == other.creator) or (@creator.nil?))
+  end
+
+  def to_s
+    "\"#{@title}\" #{@creator}/#{@date} #{@link} #{@hash}"
+  end
+end