whisperblog 0.6

data/README

@@ -0,0 +1,59 @@
+Whisper is a lightweight, simple weblaaagh server. Whisper is around 1400
+lines of code, requires no database, can serve over 1500 requests per
+second, and can output HTML, RSS, and text.
+
+    Demo: http://all-thing.net
+    News: http://all-thing.net/label/whisper/
+Homepage: http://masanjin.net/whisper/
+
+Features:
+
+- No RDBMS. Because storing your blog entries in a RDBMS is like driving to
+  work in the Space Shuttle.
+
+- YAML+Textile, sitting on a disk. Blog posts and comments are stored on
+  disk in regular files, using a mix of YAML and Textile. This means you can
+  keep your content under version control, and you can edit it with whatever
+  editor you desire.
+
+- Sits directly on top of Rack (or Thin). No intermediate layer to slow
+  things down. For benchmarks, see http://all-thing.net/whisper-benchmarks.
+
+- Lazy cached dependency graph: every bit of content is cached, built
+  lazily, and a part of a big dependency graph. That means almost every
+  request is served directly from memory, and making a change, like adding
+  or updating an entry, forces a regeneration of only those bits that
+  require it. Infrequently-requested bits of content eventually expire.
+
+- Markup enhancements: Whisper has extra processing on top of Textile to
+  syntax highlight ruby code, and turn LaTeX math expressions are turned
+  into MathML (via RiTeX). Write pretty Ruby code and pretty math without
+  any extra effort.
+
+- Fully threaded comments. Why would you not have this?
+
+- Blog post as mailing list. Comments can be made by entering your email
+  address, and replying to the resulting email. You can choose to have
+  future replies emailed to you, and replying to them automatically adds a
+  comment.  This allows you to quote, thread, and generally have a
+  reasonable discussion, which is what email is good at, and what typing
+  shit into little text areas on your web browser is not.
+
+- Multiformat support. In addition to HTML and RSS output, there’s a plain
+  text mode for the hard-core.
+
+- Pagination, labels, per-label and per-author indices, etc.
+
+Caveats:
+
+Whisper currently only supports comments via email. Textboxes are for the
+birds.
+
+Usage:
+
+1. gem install whisperblog
+2. whisper-init <blog directory>
+3. Follow the instructions!
+
+To run whisper in production mode (probably what you want for serving your
+blog), use --production.

data/bin/whisper

@@ -0,0 +1,91 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'trollop'
+require 'yaml'
+
+require "whisper"
+
+opts = Trollop.options do
+  text "Whisper v#{Whisper::VERSION}"
+
+  opt :whisper_dir, "Whisper root directory", :default => Dir.pwd
+  opt :port, "Port to run on (default: 80 in production mode, 9292 in development mode)", :short => 'p', :type => Integer
+  opt :socket, "Socket filename to run on instead of a TCP port (requires Thin)", :type => String
+  opt :development, "Development mode", :default => true, :short => :none
+  opt :production, "Production mode", :default => false, :short => :none
+  opt :log_level, "Logging level (default: 'info' in production mode, 'debug' in development mode)", :type => String
+  opt :log_file, "Log file ('-' for stdout)", :default => "-"
+  opt :console, "Run an IRB session instead of a web server", :default => false, :short => :none
+  opt :no_email, "Don't process incoming email", :default => false, :short => :none
+  conflicts :development, :production
+  conflicts :port, :socket
+end
+
+Trollop.die :log_level, "must be one of 'warn', 'info' or 'debug'" if opts[:log_level] unless %w(warn info debug).member? opts[:log_level]
+
+if opts[:production]
+  opts[:development] = false
+  opts[:log_level] ||= "info"
+else
+  opts[:development] = true
+  opts[:log_level] ||= "debug"
+end
+
+log_stream = opts[:log_file] == "-" ? $stdout : File.open(opts[:log_file], "w")
+logger = Whisper::Logger.new log_stream, opts[:log_level]
+
+config_fn = File.join opts[:whisper_dir], "config.yaml"
+config = begin
+  Whisper::Config.new config_fn, opts[:whisper_dir], opts[:development] ? :development : :production
+rescue SystemCallError => e
+  Trollop::die "Can't load configuration file #{config_fn}. Do you need to specify --whisper-dir, or run whisper-init?"
+rescue Whisper::Config::Error => e
+  Trollop::die "Error loading configuration file #{config_fn}: #{e.message}"
+end
+
+if opts[:socket]
+  Trollop::die :socket, "cannot be specified if rack_handler configuration is not 'Thin'" unless config.rack_handler == "Thin"
+end
+
+whisper = Whisper::init config
+server = Whisper::Server.new whisper[:blog], config.mode, config.static_dir, config.x_accel_redirect
+
+if opts[:console]
+  @whisper = whisper
+  @server = server
+
+  require 'irb'
+  IRB.start __FILE__
+  exit
+end
+
+app = if config.mode == :development
+  Thread.abort_on_exception = true
+  Whisper::Dependency.min_refresh_interval = 1
+  Whisper::TimedMap.expiration_interval = 1
+  Whisper::DirScanner.min_scan_interval = 1
+  Whisper::CachedFile.min_scan_interval = 1
+
+  Rack::Builder.new do
+    use Rack::CommonLogger, $stdout
+    use Rack::ShowExceptions
+    use Rack::Lint
+    run server
+  end.to_app
+else
+  whisper[:sender].start!
+  whisper[:receiver].start! unless opts[:no_email]
+
+  server
+end
+
+handler = Rack::Handler::get config.rack_handler.downcase
+run_opts = if opts[:socket]
+  { :Host => File.expand_path(opts[:socket]) }
+else
+  { :Port => (opts[:port] || config.use_this_port) }
+end
+
+handler.run app, run_opts

data/bin/whisper-init

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'yaml'
+require 'fileutils'
+require "whisper"
+
+def die s; puts s; exit end
+def mkdir d
+  FileUtils.mkdir d
+  puts "+ created: #{d}"
+end
+def copy f, t
+  FileUtils.cp f, t
+  puts "+ created: #{t}/#{File.basename(f)}"
+end
+
+dir = ARGV.shift or die("Expecting one argument: the directory to create")
+mkdir dir
+
+%w(templates static comments drafts posts authors).each { |d| mkdir File.join(dir, d) }
+share_dir = File.join File.dirname(__FILE__), "..", "share", "whisper"
+
+%w(config.yaml helper.rb formatter.rb).each { |f| copy File.join(share_dir, f), dir }
+%w(style.css mootools.js rss-badge.png spinner.gif).each { |f| copy File.join(share_dir, f), File.join(dir, "static") }
+
+template_dir = File.join(dir, "templates")
+%w(rhtml rtxt rrss).map { |ext| Dir[File.join(share_dir, "*.#{ext}")] }.flatten.each do |f|
+  copy f, template_dir
+end
+
+puts <<EOS
+Directory setup is complete. Remaining steps:
+ 1. Edit #{File.join dir, "config.yaml"} to configure your weblaaagh.
+ 2. Run "whisper-post -w #{dir}" to add a new post.
+ 3. Run "whisper -w #{dir}" and see blog at http://localhost:9292.
+ 4. To customize, edit #{template_dir}/*, helper.rb and formatter.rb.
+ 5. Have fun.
+EOS

data/bin/whisper-post

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require 'yaml'
+require 'readline'
+require 'fileutils'
+
+require "whisper"
+
+def ask q
+  ans = Readline.readline q
+  Readline::HISTORY.push ans
+  ans
+end
+
+def ask_complete q, comps
+  Readline.completion_append_character = nil
+  Readline.completion_proc = lambda { |prefix| comps.select { |c| c[0 ... prefix.length] == prefix } }
+  ans = Readline.readline q
+  Readline::HISTORY.push ans
+  ans
+end
+
+opts = Trollop.options do
+  text "Whisper v#{Whisper::VERSION}"
+  opt :whisper_dir, "Whisper root directory", :default => "."
+end
+
+config_fn = File.join opts[:whisper_dir], "config.yaml"
+begin
+  config = Whisper::Config.new config_fn, opts[:whisper_dir]
+rescue SystemCallError
+  Trollop::die "can't read config file #{config_fn}. Do you need --whisper-dir?"
+end
+
+tags = []
+authors = []
+ids = []
+Dir[File.join(config.post_dir, "*#{Whisper::ENTRY_METADATA_EXTENSION}")].each do |fn|
+  yaml = YAML.load_file(fn)
+  tags += yaml[:labels] if yaml[:labels]
+  authors << yaml[:author]
+  ids << yaml[:id]
+end
+cur_tags = tags.flatten.uniq
+cur_authors = authors.flatten.uniq
+
+title = ask "Post title: "
+default_id = title.downcase.gsub(/[^\w0-9\.]+/, "-").gsub(/\-+$/, "")
+id = ask_complete "Post id (will be used for url) [#{default_id}]: ", ids
+id = default_id if id.empty?
+tags = ask_complete("Tags (space-separated, tab for completion): ", cur_tags).split(/\s+/)
+author = ask_complete "Author email address (in \"Name <address@domain.tld>\" format, tab for completion): ", cur_authors until author && author.email_name && author.email_address
+
+fn = id
+real_fn = while true
+  real_fn = fn + Whisper::ENTRY_METADATA_EXTENSION
+  break real_fn unless File.exist? real_fn
+  fn += "-again"
+end
+
+yaml_fn = File.join config.draft_dir, real_fn
+puts "Writing #{yaml_fn}..."
+File.open(yaml_fn, "w") do |f|
+  f.print({ :author => author, :published => Time.now, :updated => Time.now,
+    :id => id, :title => title, :labels => tags }.to_yaml)
+end
+
+textile_fn = File.join config.draft_dir, (fn + Whisper::ENTRY_CONTENT_EXTENSION)
+editor = ENV["EDITOR"] || "/bin/vi"
+puts "Editing #{textile_fn} with #{editor}..."
+system "#{editor} #{textile_fn}"
+
+puts "Done!"
+
+if Readline.readline("Edit metadata (y/N)? ") =~ /^[Yy]$/
+  puts "Editing #{yaml_fn} with #{editor}..."
+  system "#{editor} #{yaml_fn}"
+end
+
+if Readline.readline("Publish (Y/n)? ") =~ /^[Yy]?$/
+  FileUtils.mv textile_fn, config.post_dir
+  FileUtils.mv yaml_fn, config.post_dir
+  puts "Published to #{config.post_dir}!"
+else
+  puts "Saved as a draft."
+end

data/bin/whisper-process-email

@@ -0,0 +1,54 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'rack'
+require 'trollop'
+require 'yaml'
+
+require "whisper"
+
+opts = Trollop.options do
+  text "Whisper v#{Whisper::VERSION}"
+
+  opt :whisper_dir, "Whisper root directory", :default => Dir.pwd
+  opt :log_level, "Set log level (default: 'info')", :type => String, :default => "debug"
+  opt :send, "In addition to writing comments, resend email", :default => false
+  opt :num, "Just process this many emails", :type => Integer
+  opt :start, "Offset to start at, if not the current mbox offset", :type => Integer
+end
+
+Trollop.die :log_level, "must be one of 'warn', 'info' or 'debug'" if opts[:log_level] unless %w(warn info debug).member? opts[:log_level]
+Trollop.die "expecting no arguments" unless ARGV.empty?
+
+logger = Whisper::Logger.new $stdout, opts[:log_level]
+
+config_fn = File.join opts[:whisper_dir], "config.yaml"
+config = begin
+  Whisper::Config.new config_fn, opts[:whisper_dir], opts[:development] ? :development : :production
+rescue SystemCallError => e
+  Trollop::die "Can't load configuration file #{config_fn}. Do you need to specify --whisper-dir, or run whisper-init?"
+rescue Whisper::Config::Error => e
+  Trollop::die "Error loading configuration file #{config_fn}: #{e.message}"
+end
+
+whisper = Whisper::init config
+receiver, sender = whisper[:receiver], whisper[:sender]
+receiver.offset = opts[:start] if opts[:start]
+
+num = 0
+until receiver.done?
+  break if opts[:num] && num >= opts[:num]
+  receiver.step
+  num += 1
+end
+
+puts "Processed #{num} messages."
+
+if opts[:send]
+  puts "Have #{sender.q_size} replies to send..."
+  sender.process until sender.done?
+else
+  puts "NOT sending #{sender.q_size} scheduled reply emails."
+end
+
+puts "Done!"

data/lib/whisper.rb

@@ -0,0 +1,52 @@
+module Whisper
+  ENTRY_METADATA_EXTENSION = ".yaml"
+  ENTRY_CONTENT_EXTENSION = ".textile"
+
+  def init config
+    ## load the formatters
+    formatter_fn = File.join(config.formatter_dir, "formatter.rb")
+    begin
+      load formatter_fn
+    rescue SystemCallError
+      Logger.instance.warn "no formatter plugins file #{formatter_fn} found"
+    end
+
+    entryset = Whisper::EntrySet.new config.post_dir
+    commentset = Whisper::CommentSet.new config.comment_dir
+    router = Whisper::Router.new config.root
+    authors = Whisper::AuthorTracker.new config.author_dir
+    blog = Whisper::Blog.new config, entryset, commentset, router, authors
+    sender = Whisper::EmailSender.new entryset, commentset, router, authors, blog.template_for("entry-email", "txt"), blog.template_for("comment-email", "txt"), config
+    receiver = Whisper::EmailReceiver.new entryset, commentset, sender, authors, config.comment_mbox, config.mbox_offset_filename, config.comment_dir
+
+    blog.install_default_routes! sender
+
+    { entryset: entryset, commentset: commentset, router: router, authors: authors,
+      blog: blog, sender: sender, receiver: receiver
+    }
+  end
+
+  module_function :init
+end
+
+require "whisper/version.rb"
+require "whisper/common.rb"
+require "whisper/dir_scanner.rb"
+require "whisper/text.rb"
+require "whisper/entry.rb"
+require "whisper/comment.rb"
+require "whisper/config.rb"
+require "whisper/cached_file.rb"
+require "whisper/page.rb"
+require "whisper/server.rb"
+require "whisper/dir_set.rb"
+require "whisper/entry_set.rb"
+require "whisper/comment_set.rb"
+require "whisper/timed_map.rb"
+require "whisper/handler.rb"
+require "whisper/router.rb"
+require "whisper/blog.rb"
+require "whisper/mbox.rb"
+require "whisper/email_receiver.rb"
+require "whisper/email_sender.rb"
+require "whisper/author_tracker.rb"

data/lib/whisper/author_tracker.rb

@@ -0,0 +1,46 @@
+require "whisper/common"
+require 'thread'
+
+module Whisper
+class AuthorTracker
+  include Loggy
+
+  def initialize dir
+    @dir = dir
+    @mutex = Mutex.new
+  end
+
+  def [] name
+    @mutex.synchronize do
+      fn = fn_for name
+      begin
+        YAML.load_file fn
+      rescue SystemCallError => e
+        {}
+      end
+    end
+  end
+
+  def []= name, info
+    unless info.empty?
+      @mutex.synchronize do
+        fn = fn_for name
+        old_info = YAML.load_file(fn) rescue {}
+        info = old_info.merge info
+        debug "writing #{info.inspect} to #{fn}..."
+        begin
+          File.open(fn, "w") { |f| f.print info.to_yaml }
+        rescue SystemCallError => e
+          warn "can't write #{fn}: #{e.message}"
+        end
+      end
+    end
+    info
+  end
+
+private
+
+  def fn_for name; File.join(@dir, name.gsub(/[\/\.]/, "-")) + ".yaml" end
+
+end
+end

data/lib/whisper/blog.rb

@@ -0,0 +1,156 @@
+require "whisper/cached_file"
+require "whisper/page"
+require "whisper/common"
+
+module Whisper
+
+class Blog
+  include Loggy
+
+  MAX_RECENT_COMMENTS = 30
+  attr_accessor :entryset, :helper
+
+  def initialize config, entryset, commentset, router, authors
+    @config = config
+    @entryset = entryset
+    @commentset = commentset
+    @router = router
+    @authors = authors
+    @helper = CachedFile.new File.join(config.helper_dir, "helper.rb")
+    @templates = {} # we'll create these lazily
+    @paginations = {}
+  end
+
+  def title; @config.title end
+  def tagline; @config.tagline end
+
+  def get_content_for path, params, request_method; @router.get_content_for path, params, request_method end
+  def template_for name, format
+    @templates[[name, format]] ||= CachedFile.new File.join(@config.template_dir, "#{name}.r#{format}")
+  end
+
+  ## we're basically doing this by hand, but we could also make a separate
+  ## paginator object and fit it into the dependency model.
+  def paginate id, timestamp, things
+    time, pages = @paginations[id]
+    if time.nil? || time < timestamp
+      pages = things.paginate @config.page_size
+      pages[0] ||= []
+      debug "repaginated #{id.inspect}: turned #{things.size} things => #{pages.size} pages"
+      time = Time.now
+      @paginations[id] = [time, pages]
+    end
+    pages
+  end
+
+  def install_default_routes! emailer=nil
+    common_opts = { :config => @config, :router => @router, :helper => @helper, :extra_deps => [@entryset, @commentset], :entryset => @entryset }
+
+    @router.add_handler :index, :get, "/index/:page:format" do |route, page, format, params|
+      debug "index handler called for page #{page.inspect} format #{format.inspect}"
+
+      Page.new common_opts.merge({ :format => format, :url_vars => { :route => route },
+        :master_template => template_for("master", format),
+        :template => template_for("list", format) }) do
+
+        pages = paginate :entries_by_page, @entryset.timestamp, @entryset.entries
+        entries = pages[page] or raise InvalidPageError
+        comments = comments_for_entries entries
+        { :entries => entries.zip(comments), :pages => pages.size, :page => page,
+          :recent_comments => recent_comments_and_entries, :route => route
+        }
+      end
+    end
+
+    @router.add_handler :label, :get, "/label/+label/:page:format" do |route, label, page, format, params|
+      debug "label handler called for label #{label.inspect} page #{page.inspect} format #{format.inspect}"
+
+      Page.new common_opts.merge({:format => format, :url_vars => { :route => route, :label => label },
+        :master_template => template_for("master", format),
+        :template => template_for("list", format) }) do
+
+        raw_entries = @entryset.entries_by_label[label] or raise InvalidPageError
+        pages = paginate [:entries_by_label, label], @entryset.timestamp, raw_entries
+        entries = pages[page] or raise InvalidPageError
+        comments = comments_for_entries entries
+        { :entries => entries.zip(comments), :pages => pages.size, :page => page,
+          :recent_comments => recent_comments_and_entries, :route => route, :label => label
+        }
+      end
+    end
+
+    @router.add_handler :author, :get, "/by/+author/:page:format" do |route, author, page, format, params|
+      debug "author handler called for author #{author.inspect} page #{page.inspect} format #{format.inspect}"
+
+      Page.new common_opts.merge({:format => format, :url_vars => { :route => route, :author => author },
+        :master_template => template_for("master", format),
+        :template => template_for("list", format) }) do
+
+        pages = paginate [:entries_by_author, author], @entryset.timestamp, @entryset.entries_by_author[author]
+        entries = pages[page] or raise InvalidPageError
+        comments = comments_for_entries entries
+        { :entries => entries.zip(comments), :pages => pages.size, :page => page,
+          :recent_comments => recent_comments_and_entries, :route => route, :author => author
+        }
+      end
+    end
+
+    ## allow formats on /. why not.
+    @router.add_handler :root, :get, "/:format" do |route, format, params|
+      debug "root handler called with format #{format}"
+      Page.new common_opts.merge({ :format => format, :url_vars => { :route => :index },
+        :template => template_for("list", format),
+        :master_template => template_for("master", format) }) do
+
+        pages = paginate :entries_by_page, @entryset.timestamp, @entryset.entries
+        entries = pages[0]
+        comments = comments_for_entries entries
+        { :entries => entries.zip(comments), :pages => pages.size, :page => 0,
+          :recent_comments => recent_comments_and_entries, :route => route
+        }
+      end
+    end
+
+    if emailer
+      @router.add_handler :comment, :post, "/comment/+id" do |route, id, params|
+        debug "request comment handler called for id #{id} with params #{params.inspect}"
+        begin
+          emailer.handle_send_request id, params if emailer
+          ["email sent!", "text/plain"]
+        rescue EmailSender::Error => e
+          ["error: #{e.message}", "text/plain"]
+        end
+      end
+    end
+
+    ## individual entry handler is the most generic (entry ids can be close to
+    ## anything), so we install it last.
+    @router.add_handler :entry, :get, "/+id:format" do |route, id, format, params|
+      debug "entry handler called for id #{id.inspect} format #{format.inspect}"
+
+      Page.new common_opts.merge({ :format => format, :url_vars => { :route => route, :id => id },
+        :template => template_for("entry", format),
+        :master_template => template_for("master", format)
+        }) do
+
+        raise InvalidPageError unless (entry = @entryset.entries_by_id[id])
+        { :entry => entry, :comments => (@commentset.comment_tree_by_entry_id[entry.id] || {}),
+          :recent_comments => recent_comments_and_entries, :route => route,
+          :author_info => @authors
+        }
+      end
+    end
+  end
+
+  def recent_comments_and_entries
+    comments = @commentset.comments[0 ... MAX_RECENT_COMMENTS]
+    entries_per_comment = @entryset.entries_by_id.values_at(*comments.map { |c| c.entry_id })
+    comments.zip entries_per_comment
+  end
+
+  def comments_for_entries entries
+    @commentset.comments_by_entry_id.values_at(*entries.map { |e| e.id }).map { |x| x || [] }
+  end
+end
+
+end