RubyGems - mlmmj-rbarchiver - Versions diffs - 0.0.1 - Mend

mlmmj-rbarchiver 0.0.1

Files changed (12) hide show

checksums.yaml +7 -0
data/COPYING +674 -0
data/README.md +151 -0
data/bin/mlmmj-rbarchiver +102 -0
data/data/mhonarc-rc.erb +465 -0
data/extra/archive.css +81 -0
data/extra/man/mlmmj-rbarchiver.1 +144 -0
data/extra/rbarchiver.conf +50 -0
data/lib/mlmmj-archiver.rb +38 -0
data/lib/mlmmj-archiver/archiver.rb +327 -0
data/lib/mlmmj-archiver/version.rb +25 -0
metadata +110 -0

@@ -0,0 +1,81 @@
+body {
+    margin: 0px;
+    background-color: #EEEEEE;
+    font-family: serif;
+}
+a {
+    color: blue;
+}
+a:hover {
+    color: red;
+}
+/* Hide all those unnessary rules generated by MHonArc */
+hr {
+    display: none;
+}
+h1 {
+    border-bottom: 1px solid black;
+}
+div#banner {
+    position: fixed;
+    width: 100%;
+    height: 60px;
+    padding: 4px;
+    background-color: black;
+}
+div#main {
+    padding: 4px;
+}
+div#header p {
+    font-size: 14pt;
+    color: white;
+    margin: 0px 0px 4px 0px;
+}
+div#mailinglists {
+    padding-top: 60px; /* Height of banner plus distance */
+}
+div#banner a {
+    color: white;
+}
+div#banner a:hover {
+    color: red;
+}
+div#toplinks, div#topnav {
+    margin-top: 18px;
+    color: white;
+}
+ul.message-headers {
+    background-color: white;
+    list-style-type: none;
+    border: 1px dashed black;
+    padding: 4px;
+}
+ul.message-headers em {
+    font-weight: bold;
+}
+div#tidx ul {
+    list-style-type: none;
+}
+div#didx ul {
+    list-style-type: none;
+}
+blockquote.mlquote {
+    border-left: 2px solid blue;
+    padding-left: 4px;
+}

data/extra/man/mlmmj-rbarchiver.1 ADDED

@@ -0,0 +1,144 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "MLMMJ\-RBARCHIVER" "1" "February 2014" "mlmmj" "General Commands Manual"
+.
+.SH "NAME"
+\fBmlmmj\-rbarchiver\fR \- Convert mlmmj MLs to an HTML archive
+.
+.SH "SYNPOSIS"
+\fBmlmmj\-rbarchive \-i INDIR \-o OUTDIR\fR \fIOPTIONS\fR
+.
+.SH "DESCRIPTION"
+\fBmlmmj\-rbarchive\fR converts an mlmmj mailinglist into a browsable HTML archive\. You can point your webserver to the resulting directory and thereby serve it as a (public) webarchive for the mailinglist\.
+.
+.SH "USAGE"
+The \fB\-i\fR and \fB\-o\fR options are required and the program will abort if any of them is missing\. The former specifies the root directory of the mlmmj mailinglist you want to process, whereas the latter specifies the directory the HTML will be outputted to\. To be exact, the program will take the \fBarchive/\fR subdirectory of the input directory for processing and will output to a directory named after the processed ML below the output directory\.
+.
+.SH "OPTIONS"
+.
+.TP
+\fB\-a EMAIL\fR, \fB\-\-admin EMAIL\fR
+Administrative Email contact\.
+.
+.TP
+\fB\-c FILE\fR, \fB\-\-config\-file FILE\fR
+Configuration file to read options from\. See below for more information on this\.
+.
+.TP
+\fB\-C DIR\fR, \fB\-\-cachedir DIR\fR
+Cache directory for storing the sorted emails\. Usually this is set automatically to some temporary directory, but you can set this to a permanent directory if you want\. This will cause the program to not regenerate the entire ML archive, but instead use what can be found in that cache directory, resulting in a faster processing, especially if the mailinglist in question is large\.
+.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Display a short option summary\.
+.
+.TP
+\fB\-H HTML\fR, \fB\-\-header HTML\fR
+A short HTML snippet placed at the top of each page\. Set this to something like \fB<p>My Mailinglist archive</p>\fR\.
+.
+.TP
+\fB\-i DIR\fR, \fB\-\-indir DIR\fR
+The mlmmj mailinglist to process\. \fIRequired\fR\.
+.
+.TP
+\fB\-l\fR, \fB\-\-levels NUM\fR
+Maximum indentation level before the threads are flattened\.
+.
+.TP
+\fB\-m BINARY\fR, \fB\-\-mhonarc BINARY\fR
+Path to the mhonarc(1) executable\. Usually this is found out automatically\.
+.
+.TP
+\fB\-o DIR\fR, \fB\-\-outdir DIR\fR
+The target directory for the HTML files\. \fIRequired\fR\.
+.
+.TP
+\fB\-s\fR, \fB\-\-style STRING\fR
+CSS link to include in each HTML files’ \fB<head>\fR section\. This is copied as\-is, you have to provide the CSS file yourself\.
+.
+.TP
+\fB\-S\fR, \fB\-\-search URL\fR
+Add a link called "search" to the navigation bar that links to the URL specified here\. Nothing more is done, you have to implement the target yourself\.
+.
+.TP
+\fB\-v\fR, \fB\-\-version\fR
+Print the version number and exit\.
+.
+.TP
+\fB\-V\fR, \fB\-\-verbose\fR
+Give more information while processing\.
+.
+.TP
+\fB\-x\fR, \fB\-\-no\-checknoarchive\fR
+Do \fInot\fR check the \fBX\-NoArchive\fR email header that instructs archiving software to ignore an email\. Note that the default behaviour is to honour that header\.
+.
+.SH "CONFIGURATION FILE"
+The \fB\-c\fR option allows you to specify a configuration file so you don’t have to pass all the above commandline options again and again\. Instead you can set them in the file and have the program read that file on startup\. Most things specifyable via the commandline can be set in the config file too, so I encourage you to use such a file to keep your commands clean\. The mapping of the configuration file directives to the above commandline options is as follows:
+.
+.IP "" 4
+.
+.nf
+┌────────────────┬────────┐
+│ Directive      │ Option │
+├────────────────┼────────┤
+│ archiveadmin   │ \-a     │
+│ cachedir       │ \-c     │
+│ checknoarchive │ \-x     │
+│ header         │ \-H     │
+│ indir          │ \-i     │
+│ mhonarc        │ \-m     │
+│ outdir         │ \-o     │
+│ tlevels        │ \-l     │
+│ searchtarget   │ \-S     │
+│ stylefile      │ \-s     │
+└────────────────┴────────┘
+.
+.fi
+.
+.IP "" 0
+.
+.P
+See the example configuration file provided in the \fBextra/\fR directory in the source for concrete usage instructions\.
+.
+.SH "EXAMPLE"
+Suppose you want to process the mlmmj mailinglist at \fB/var/spool/mlmmj/mymailinglist\fR and output the HTML to the directory \fB/var/www/mailarchive/mymailinglist\fR\. This can be achieved as follows:
+.
+.IP "" 4
+.
+.nf
+$ mlmmj\-rbarchive \-i /var/spool/mlmmj/mymailinglist \-o /var/www/mailarchive
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Note you don’t have to specify the ML subdirectory, this is created automatically for you\.
+.
+.P
+The resulting directory \fB/var/www/mailarchive/mymailinglist\fR will contain all messages sent to the ML converted to HTML\. Note this does \fInot\fR create a toplevel \fBindex\.html\fR file for you, so you probably want to create one that contains information about how to subscribe/unsubscribe from the mailinglists archived at the website\. That file will be left alone by \fBmlmmj\-rbarchive\fR\.
+.
+.P
+An example cron(8) entry for periodically running that command could look like this:
+.
+.IP "" 4
+.
+.nf
+0 2 * * * mlmmj\-rbarchive \-i /var/spool/mymailinglist \-o /var/www/mailarchive
+.
+.fi
+.
+.IP "" 0
+.
+.P
+That would run the shown command every day at 02:00 in the night\.
+.
+.SH "AUTHOR"
+Marvin Gülker \fB<quintus@quintilianus\.eu>\fR
+.
+.SH "SEE ALSO"
+mlmmj website (\fIhttp://www\.mlmmj\.org\fR), project page (\fIhttps://github\.com/Quintus/mlmmj\-rbarchive\fR)

data/extra/rbarchiver.conf ADDED

@@ -0,0 +1,50 @@
+# -*- conf-colon -*-
+# mlmmj-rbarchiver example configuration file.
+#
+# Values are set to its default ones unless noted otherwise.
+# Delete options you don't change from your config to keep
+# it clean if you want.
+# REQUIRED option. Set this to the mailinglist you want
+# to archive.
+indir: /var/spool/mlmmj/mymailinglist
+# REQUIRED option. Set this to the directory you want to
+# output the HTML files to.
+outdir: /var/www/mailinglists/mymailinglist
+# Directory where the mails are stored sorted in.
+# Usually you don't want to set this, but setting this
+# to a permanent storage will speed up the archiving
+# process on large MLs.
+#cachedir: /tmp/mlmmjarchiver-tmp
+# Short HTML snippet shown at the top of each page.
+header: <p>ML archive</p>
+# If set to "yes", mails with the X-NoArchive header set
+# will not be included into the webarchive. "no" archives
+# them nevertheless.
+checknoarchive: yes
+# Maximum depth of a message tree before it is flattened.
+tlevels: 8
+# Administrative email shown in the footer.
+archiveadmin: postmaster@example.org
+# This is copied as-is into a CSS link in the <head> section
+# of each page. You want to use this to style the pages the
+# way you want them.
+stylefile: /archive.css
+# If this is set, displays a link "search" at the top of
+# each ML page that links to the location specified by
+# this option. Note you do have to implement everyting
+# there yourself!
+#searchtarget: /search
+# Path to the `mhonarc' executable. Usually you won't need
+# to specify this. A relative path makes mlmmj-rbarchiver
+# search the PATH environment variable.
+#mhonarc: mhonarc

data/lib/mlmmj-archiver.rb ADDED

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# This file is part of mlmmj-rbarchive.
+#
+# mlmmj-rbarchive makes a web archive from your mlmmj-archive.
+# Copyright (C) 2013  Marvin Gülker
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+require "tempfile"
+require "fileutils"
+require "pathname"
+require "erb"
+require "mail"
+require "paint"
+begin
+  require "rb-inotify"
+rescue LoadError
+end
+# Namespace for this library.
+module MlmmjArchiver
+end
+require_relative "mlmmj-archiver/version"
+require_relative "mlmmj-archiver/archiver"

data/lib/mlmmj-archiver/archiver.rb ADDED

@@ -0,0 +1,327 @@
+# -*- coding: utf-8 -*-
+# This file is part of mlmmj-rbarchive.
+#
+# mlmmj-rbarchive makes a web archive from your mlmmj-archive.
+# Copyright (C) 2013-2014  Marvin Gülker
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# Archiver class. Point it to a target directory you want to place your web
+# archive under, add some MLs to process and start the process via #archive!.
+# You have some influence over the used (temporary) MHonArc RC file by specifying
+# some arguments to ::new.
+#
+# Note that archiving for the web is a two-step process. First the mails in
+# mlmmj’s +archive+ folder need to be split up in a directory structure that
+# allows processesing them month-by-month instead of processing them all at once,
+# because this allows for an easier overview of the web archive. In the second
+# step, all these month directories are passed into +mhonarc+, which converts
+# them to HTML and stores them in the final directory.
+class MlmmjArchiver::Archiver
+  # Path relative to ML root containing the mails
+  ARCHIVE_DIR  = "archive"
+  # Path relative to ML root containing the file that
+  # requests the web archiving.
+  CONTROL_FILE = "control/webarchive".freeze
+  # Path to the +mhonarc+ executable.
+  MHONARC = "/usr/bin/mhonarc"
+  # Default values for the MHonArc RC file.
+  MRC_DEFAULTS = {
+    :header => "<p>ML archive</p>",
+    :tlevels => 8,
+    :archiveadmin => "postmaster@example.org",
+    :checknoarchive => true,
+    :searchtarget => nil,
+    :stylefile => "/archive.css"
+  }.freeze
+  # Template for generating the temporary MHonArc RC file.
+  MRC_TEMPLATE = ERB.new(File.read(File.join(File.expand_path(File.dirname(__FILE__)), "..", "..", "data", "mhonarc-rc.erb")))
+  # Create a new Archiver that stores its HTML mails below
+  # the given +target+ directory. +rc_args+ allows
+  # the customization of the used MHonArc RC file.
+  # It is a hash that takes the following arguments
+  # (the values in parentheses denote the default values)
+  # [header ("<p>ML archive</p>")]
+  #   HTML header to prepend to every page. $IDXTITLE$ is replaced
+  #   by the title of the respective index.
+  # [tlevels (8)]
+  #   Number of levels to nest threads before flattening.
+  # [archiveadmin (postmaster@example.org)]
+  #   E-Mail address of the archive administrator.
+  # [checknoarchive (true)]
+  #  If set, adds <CHECKNOARCHIVE> to the rc file. Otherwise
+  #  adds <NOCHECKNOARCHIVE>.
+  # [searchtarget (nil)]
+  #   If this is set, displays a link called "search" next to
+  #   the index links that links to the location specified here.
+  # [stylefile ("/archive.css")]
+  #   CSS style file to reference from the outputted HTML pages.
+  # [mhonarc ("/usr/bin/mhonarc")]
+  #   Path to the +mhonarc+ executable to create the archive.
+  # [cachedir (nil)]
+  #  Path to a directory where the mails are stored sorted.
+  #  Setting this to a permanent storage will speed up the
+  #  archiving process on large MLs.
+  def initialize(target, rc_args = {})
+    @target_dir   = Pathname.new(target).expand_path
+    @mailinglists = []
+    @mutex        = Mutex.new
+    @rc_args      = MRC_DEFAULTS.merge(rc_args)
+    @debug        = false
+    @inotify_thread = nil
+    @mhonarc      = rc_args[:mhonarc] || MHONARC
+    if rc_args[:cachedir]
+      @sorted_target = Pathname.new(rc_args[:cachedir]).expand_path
+    else
+      @sorted_target = Pathname.new(Dir.mktmpdir)
+      at_exit{FileUtils.rm_rf(@sorted_target)}
+    end
+  end
+  # Enable/disable debugging output.
+  def debug_mode=(val)
+    @debug = val
+  end
+  # True if debugging output is enabled, see #debug_mode=.
+  def debug_mode?
+    @debug
+  end
+  # Add a mlmmj ML directory to process.
+  def add_ml(path)
+    dir = Pathname.new(path).expand_path
+    debug("Adding ML directory: #{dir}")
+    @mailinglists.push(dir)
+  end
+  # Like #add_ml, but returns +self+ for method chaining.
+  def <<(path)
+    add_ml(path)
+    self
+  end
+  # The more elegant variant of #preprocess_mlmmj_mails. Instead of polling all
+  # mails and testing whether they are there, use inotify to have Linux notify
+  # us when a new file is added to the ML directory. For this method to work
+  # +rb-inotify+ must be available on your system (otherwise you get a
+  # NotImplementedError).
+  def watch_mlmmj_mails!
+    raise(NotImplementedError, "This is only possible with rb-inotify!") unless defined?(INotify)
+    @inotifier = INotify::Notifier.new
+    @mailinglists.each do |path|
+      archive_dir = path + ARCHIVE_DIR
+      @inotifier.watch(archive_dir.to_s, :create) do |event|
+        next unless File.file?(event.absolute_name)
+        next unless event.name =~ /^\d+$/
+        debug "Got a new mail: #{event.name}"
+        sleep 2 # Wait for the file to be fully written
+        @mutex.synchronize do
+          mail = Mail.read(event.absolute_name)
+          FileUtils.cp(event.absolute_name, @sorted_target + path.basename + mail.date.year.to_s + mail.date.month.to_s)
+        end
+      end
+    end
+    debug "Watching MLs via inotify."
+    @inotify_thread = Thread.new{@inotifier.run}
+  end
+  # Terminate the watching thread started by #watch_mlmmj_mails.
+  def stop_watching_mlmmj_mails!
+    @inotify_thread.terminate
+  end
+  # Iterates over all mailinglists and copies new messages into
+  # the intermediate month directory structure.
+  def preprocess_mlmmj_mails!
+    @sorted_target.mkpath unless @sorted_target.directory?
+    @mutex.synchronize do
+      @mailinglists.each do |path|
+        hsh = collect_messages(path + ARCHIVE_DIR)
+        split_messages_into_month_dirs(hsh, @sorted_target + path.basename) # path.basename is the ML name
+      end
+    end
+  end
+  # Process all the mails in all the directories.
+  def archive!
+    @mutex.synchronize do
+      rcpath = generate_rcfile
+      @mailinglists.each do |path|
+        control_file = path + CONTROL_FILE
+        next unless control_file.file?
+        process_ml(@sorted_target + path.basename, @target_dir + path.basename, rcpath)
+      end
+    end
+  end
+  # Search the given mailinglist for a specific search term.
+  # Return value is an array of paths relative to the HTML
+  # directory of the given ML. +query+ may be a regular
+  # expression or simply a string to check for.
+  def search(mlname, query)
+    html_dir = @target_dir + mlname
+    results = []
+    html_dir.find do |path|
+      next unless path.file?
+      next unless path.basename.to_s =~ /^\d+\.html$/
+      # Check if the file content matches
+      content = File.read(path)
+      if query.kind_of?(Regexp)
+        result = content =~ query
+      else
+        result = content.downcase.include?(query.downcase)
+      end
+      # If it did, remember it for returning
+      results << path.relative_path_from(html_dir) if result
+    end
+    results
+  end
+  private
+  # [header ("<p>ML archive</p>")]
+  #   HTML header to prepend to every page. $IDXTITLE$ is replaced
+  #   by the title of the respective index.
+  # [tlevels (8)]
+  #   Number of levels to nest threads before flattening.
+  # [archiveadmin (postmaster@example.org)]
+  #   E-Mail address of the archive administrator.
+  # [checknoarchive (true)]
+  #  If set, adds <CHECKNOARCHIVE> to the rc file. Otherwise
+  #  adds <NOCHECKNOARCHIVE>.
+  # [searchtarget ("/search")]
+  #   Target for the "search" link.
+  # [stylefile ("/archive.css")]
+  # Generate an RC file for MHonArc and return the path to it.
+  def generate_rcfile
+    tempfile = Tempfile.new("archive-mhonarc")
+    rcpath   = tempfile.path
+    at_exit{File.delete(rcpath)}
+    debug "Generating MhonArc RC file at #{rcpath}"
+    header         = @rc_args[:header]
+    tlevels        = @rc_args[:tlevels]
+    archiveadmin   = @rc_args[:archiveadmin]
+    checknoarchive = @rc_args[:checknoarchive] ? "<CHECKNOARCHIVE>" : "<CHECKNOARCHIVE>\n<NOCHECKNOARCHIVE>"
+    searchtarget   = @rc_args[:searchtarget]
+    stylefile      = @rc_args[:stylefile]
+    mrc = MRC_TEMPLATE.result(binding)
+    tempfile.write(mrc)
+    rcpath
+  end
+  # Process all mails in +sorted_mail_dir+ and output an HTML
+  # directory structure in +archive_dir+. +rcpath+ is the
+  # path to an MHonArc RC file to use.
+  def process_ml(sorted_mail_dir, archive_dir, rcpath)
+    debug "Processing sorted ML directory #{sorted_mail_dir} ===> #{archive_dir}"
+    # Create the target directory
+    archive_dir.mkpath unless archive_dir.directory?
+    # Let mhonarc process the messages
+    sorted_mail_dir.each_child do |yeardir|
+      yeardir.each_child do |monthdir|
+        mhonarc(monthdir, archive_dir + sprintf("%04d/%02d", yeardir.basename.to_s.to_i, monthdir.basename.to_s.to_i), rcpath)
+      end
+    end
+  end
+  # Collect the mails in the given directory in a nested hash like this:
+  #   {year1 => {month1 => [...], month2 => [...]}, year2 => {...}}
+  def collect_messages(mail_dir)
+    hsh = Hash.new{|hsh, k| hsh[k] = Hash.new{|hsh2, k2| hsh2[k2] = []}}
+    debug "Collecting messages in #{mail_dir}"
+    mail_dir.each_child do |path|
+      next unless path.file?
+      mail = Mail.read(path)
+      hsh[mail.date.year][mail.date.month] << path
+    end
+    hsh
+  end
+  # Takes the result of #collect_messages and writes the messages
+  # out to a directory structure under +target+ like this:
+  #   2013/
+  #     1/
+  #       msg1
+  #     2/
+  #       msg1
+  #       msg2
+  #   ...
+  # Already existing messages will not be copied again.
+  def split_messages_into_month_dirs(hsh, target)
+    debug "Splitting into year-month directories under #{target}"
+    target.mkpath unless target.directory?
+    hsh.each_pair do |year, months|
+      year_dir = target + year.to_s
+      year_dir.mkdir unless year_dir.directory?
+      months.each do |month, messages|
+        month_dir = year_dir + month.to_s
+        month_dir.mkdir unless month_dir.directory?
+        messages.each do |msgpath|
+          FileUtils.cp(msgpath, month_dir) unless month_dir.join(msgpath.basename).file?
+        end
+      end
+    end
+  end
+  # Run mhonarc over the +source+ directory and place the
+  # results in +rel_target+ which is a path relative to
+  # the +target+ passed to ::new. +rcpath+ is the path to
+  # an MHonArc RC file to use.
+  def mhonarc(source, rel_target, rcpath)
+    target = @target_dir + rel_target
+    target.mkpath unless target.directory?
+    ary = [@mhonarc.to_s, "-rcfile", rcpath.to_s, "-outdir", target.to_s, "-add", source.to_s]
+    debug "Executing: #{ary.inspect}"
+    system(*ary)
+  end
+  # Prints +str+ onto stdout via #puts if #debug_mode?.
+  def debug(str)
+    puts str if debug_mode?
+  end
+end