whisperblog 0.6

data/lib/whisper/dir_scanner.rb

@@ -0,0 +1,76 @@
+require 'find'
+require "whisper/common"
+
+module Whisper
+
+## watches a directory tree, keeps a list of filenames, reports modifications,
+## additions and removals. scans at most every +min_scan_interval+ seconds.
+##
+## TODO: rename to cacheddir
+class DirScanner
+  include Loggy
+  include Dependency
+
+  class << self; attr_accessor :min_scan_interval end
+
+  def initialize path, filename_regex
+    @path = path
+    @filename_regex = filename_regex
+    @last_scan = Time.at 0
+
+    @files = {}
+    @added, @removed, @updated = [], [], []
+
+    DirScanner.min_scan_interval ||= 30
+    dependency_init
+  end
+
+  def dependencies; [] end
+
+  attr_reader :path, :timestamp
+
+  ## returns the diffs since the last time diff was called
+  def diff; content end
+
+  def build old
+    #debug "recording changes for myself"
+    rescan!
+    added, removed, updated = @added, @removed, @updated
+    added.each { |fn| @files[fn] = true }
+    removed.each { |fn| @files.delete fn }
+    @added, @removed, @updated = [], [], []
+    [added, removed, updated]
+  end
+
+  def check; rescan! end
+
+  def rescan!
+    return false unless @last_scan + DirScanner.min_scan_interval < Time.now
+    @added, @removed, @updated = [], [], []
+
+    current = {}
+    Find.find(@path) do |f|
+      next unless f =~ @filename_regex
+      current[f] = true
+    end
+
+    current.keys.each do |fn|
+      if @files[fn]
+        @updated << fn if File.mtime(fn) > @last_scan
+        next
+      end
+      @added << fn
+    end
+
+    @files.keys.each do |fn|
+      next if current[fn]
+      @removed << fn
+    end
+
+    debug "found #{@added.size} added, #{@removed.size} removed, and #{@updated.size} modified files in #{@path}"
+    @last_scan = Time.now
+    !@added.empty? || !@removed.empty? || !@updated.empty?
+  end
+end
+
+end

data/lib/whisper/dir_set.rb

@@ -0,0 +1,50 @@
+require "whisper/common"
+require "whisper/dir_scanner"
+
+module Whisper
+
+## maintains a set of things from a directory, where each thing is has both a
+## .yaml and .textile file. 
+class DirSet
+  include Loggy
+  include Dependency
+
+  def initialize dir, klass
+    @scanner = DirScanner.new dir, /(#{Regexp.escape ENTRY_METADATA_EXTENSION}|#{Regexp.escape ENTRY_CONTENT_EXTENSION})$/
+    @klass = klass
+
+    dependency_init
+  end
+
+  def dependencies; [@scanner] end
+  def things; content end
+  def size; things.size end
+
+private
+
+  def build items
+    items ||= []
+    added, deleted, modified = @scanner.diff
+
+    added.each do |fn|
+      next unless fn =~ /#{Regexp::escape ENTRY_METADATA_EXTENSION}$/
+      content_fn = fn.sub(/#{Regexp::escape ENTRY_METADATA_EXTENSION}$/, Whisper::ENTRY_CONTENT_EXTENSION)
+      unless File.exist?(content_fn)
+        warn "found yaml file but no textile file for #{content_fn}"
+        next
+      end
+
+      debug "adding new item: #{fn}"
+      items << @klass.new(CachedFile.new(fn), CachedFile.new(content_fn))
+    end
+
+    deleted.each do |fn|
+      debug "deleting item: #{fn}"
+      items.delete_if { |e| e.meta_file.path == fn }
+    end
+
+    items
+  end
+end
+
+end

data/lib/whisper/email_receiver.rb

@@ -0,0 +1,212 @@
+require 'thread'
+require 'set'
+require 'time'
+require 'digest/md5'
+require "whisper/common"
+require "whisper/mbox"
+require "whisper/comment"
+
+module Whisper
+
+class EmailReceiver
+  include Loggy
+
+  DELAY = 20
+
+  def initialize entryset, commentset, email_sender, authors, mbox_fn, offset_fn, comment_dir
+    @entryset = entryset
+    @commentset = commentset
+    @email_sender = email_sender
+    @offset_fn = offset_fn
+    @authors = authors
+    @comment_dir = comment_dir
+    offset = IO.read(offset_fn).to_i rescue 0
+    @mbox_fn = mbox_fn
+    @mbox = Mbox.new mbox_fn, offset
+  end
+
+  def offset= offset
+    @mbox = Mbox.new @mbox_fn, offset
+  end
+
+  def start!
+    Thread.new do
+      while true
+        begin
+          sleep DELAY
+          step
+        rescue Exception => e
+          warn ["(#{e.class}) #{e.message}", e.backtrace].flatten.join("\n")
+        end
+      end
+    end
+  end
+
+  def done?; @mbox.eof? end
+
+  def step
+    return false if @mbox.eof?
+
+    content, vars, entry, parent_comment = process_one_message
+    if content
+      comment = write_one_message content, vars
+      resend_comments comment, parent_comment, entry
+    end
+    update_offset!
+    true
+  end
+
+  ## resend a new comment to everyone who should get it
+  def resend_comments comment, parent_comment, entry
+    sent_to = Set.new [comment.author.email_address] # never send the comment back to the original author
+
+    if parent_comment
+      addr = parent_comment.author.email_address
+      resend = parent_comment.resend_setting
+      debug "for parent comment, resend for #{addr} is #{resend.inspect}"
+      if resend == "replies-only"
+        info "scheduling a resend of #{comment.id} to #{addr} because they're the author of the parent comment and resend is #{resend}"
+        sent_to << addr
+        @email_sender.send_comment comment, addr
+      end
+    else
+      ## resend to entry author if it's a top-level comment
+      info "scheduling a resend of top-level comment #{comment.id} to entry author #{entry.author.email_address}"
+      @email_sender.send_comment comment, entry.author.email_address
+    end
+
+    (@commentset.comments_by_entry_id[entry.id] || []).each do |c| # iterate over all comments for the entry
+      addr = c.author.email_address
+      next if sent_to.member? addr # we've already seen him
+      sent_to << addr
+      resend = c.resend_setting
+      debug "for thread, resend for #{addr} comment #{c.id} is #{resend.inspect}"
+      if resend == "all"
+        info "scheduling a resend of #{comment.id} to #{addr} because they're in the thread and resend is #{resend}"
+        @email_sender.send_comment comment, addr
+      end
+    end
+  end
+
+  def process_one_message
+    offset = @mbox.offset
+    body, headers = @mbox.next_message
+    return unless body
+
+    message_id, reply_to, from, date = %w(message-id in-reply-to from date).map do |f|
+      headers[f] or begin
+        warn "no #{f} for message at offset #{offset}. have headers: #{headers.keys.inspect}"
+        return
+      end
+    end
+
+    if message_id =~ /^<(.+?)>$/
+      message_id = $1
+    end
+
+    if reply_to =~ /^<(.+?)>$/
+      reply_to = $1
+    end
+
+    @commentset.refresh! # force comment set to refresh itself before we look for parents
+    entry, parent_comment = case reply_to
+    when /whisper-post-(\S+)@/
+      unless(entry = @entryset.entries_by_id[$1])
+        warn "invalid entry id in message-id for message at offset #{offset}: #{$1.inspect}"
+        return
+      end
+      debug "found new message from #{headers['from']} on entry #{entry.id}"
+      [entry, nil]
+    else
+      unless(parent_comment = @commentset.comments_by_message_id[reply_to])
+        warn "no such comment id in in-reply-to for message at offset #{offset}: #{reply_to.inspect}"
+        return
+      end
+      unless(entry = @entryset.entries_by_id[parent_comment.entry_id])
+        warn "comment #{comment.id} doesn't reference a real entry? wtf (offset #{offset})"
+        return
+      end
+      debug "found new message from #{headers['from']} on entry #{entry.id} replying to a comment by #{parent_comment.author}"
+      [entry, parent_comment]
+    end
+
+    ## pull out variables and comments
+    author_vars = {}
+    content, author_vars = pull_out_content_and_vars body
+    @authors[from.email_address] = author_vars
+
+    vars = { :author => from, :offset => offset, :entry_id => entry.id, :message_id => message_id,
+             :resend_setting => (author_vars[Comment::RESEND_SETTING] || @authors[from.email_address][Comment::RESEND_SETTING]),
+             :url_setting => (author_vars[Comment::URL_SETTING] || @authors[from.email_address][Comment::RESEND_SETTING]),
+             :published => Time.parse(date), :parent_id => (parent_comment ? parent_comment.id : nil) }
+
+    [content, vars, entry, parent_comment]
+  end
+
+  def write_one_message content, vars
+    id = Digest::MD5.hexdigest [vars[:author], vars[:entry_id], vars[:message_id]].join("|")
+
+    dir = File.join @comment_dir, vars[:entry_id]
+    Dir.mkdir dir unless File.exist? dir
+
+    name = File.join dir, id
+    yaml_fn = name + ENTRY_METADATA_EXTENSION
+    warn "overwriting comment yaml file: #{yaml_fn}" if File.exist? yaml_fn
+    File.open(yaml_fn, "w") { |f| f.puts vars.merge({ :id => id }).to_yaml }
+
+    textile_fn = name + ENTRY_CONTENT_EXTENSION
+    warn "overwriting comment textile file: #{textile_fn}" if File.exist? textile_fn
+    File.open(textile_fn, "w") { |f| f.print content }
+
+    info "wrote comment #{yaml_fn}"
+
+    Comment.new CachedFile.new(yaml_fn), CachedFile.new(textile_fn)
+  end
+
+  def pull_out_content_and_vars body
+    vars = {}
+    last_was_comment = false
+    attribution = nil
+
+    lines = body.gsub(/^-- \n.*\Z/m, "").lines.to_a
+    content = lines.zip(lines[1 .. -1] || []).map do |l, next_l|
+      case l
+      when /^[^>].*:$/
+        if next_l =~ /^> ;/
+          ## top-level attribution followed by comments. may need to consume it.
+          # puts "have attribution: #{l}"
+          attribution = l
+          nil
+        else
+          l
+        end
+      when /^> ! (.*?):\s*(.*?)(\s*;.*)?$/
+        vars[$1] = $2
+        nil
+      when /^> ;/
+        last_was_comment = true
+        nil
+      when /^>\s*$/
+        last_was_comment ? nil : l
+      when /^> /
+        if attribution
+          both = [attribution, l]
+          attribution = nil
+          both
+        else
+          l
+        end
+      else
+        l
+      end
+    end.flatten.compact.join
+    [content, vars]
+  end
+
+  def update_offset!
+    File.open(@offset_fn, "w") { |f| f.puts @mbox.offset }
+  end
+end
+
+end
+

data/lib/whisper/email_sender.rb

@@ -0,0 +1,114 @@
+require 'erb'
+require 'socket'
+require 'thread'
+require "whisper/common"
+require "whisper/timed_map"
+
+module Whisper
+
+class EmailSender
+  class Error < StandardError; end
+
+  include Loggy
+
+  def initialize entryset, commentset, router, authors, entry_template, comment_template, config
+    @entryset = entryset
+    @commentset = commentset
+    @authors = authors
+    @router = router
+    @entry_template = entry_template
+    @comment_template = comment_template
+    @config = config
+    @hostname = Socket.gethostname
+    @map = TimedMap.new
+    @q = Queue.new
+  end
+
+  def done?; @q.empty? end
+  def q_size; @q.size end
+
+  def start!
+    Thread.new do
+      while true
+        begin
+          process
+        rescue Exception => e
+          warn e.message
+          warn e.backtrace.join("\n")
+        end
+      end
+    end
+  end
+
+  def from_address; @config.from_email_address end
+  def blog_title; @config.title end
+  def full_url_for(*a); @config.public_url_root + @router.url_for(*a) end
+
+  def process
+    thing, to_address, web_request = @q.pop
+
+    @map.prune!
+    content, timestamp = @map[thing.id]
+
+    if timestamp.nil? || @entry_template.timestamp > timestamp || @comment_template.timestamp > timestamp
+      content = case thing
+        when Entry; get_content_for_entry thing, to_address, web_request
+        when Comment; get_content_for_comment thing, to_address, web_request
+        else raise "wtf: #{thing.inspect}"
+      end
+      @map[thing.id] = [content, timestamp]
+    end
+
+    debug "sending content to #{to_address} from #{from_address} via #{@config.sendmail}"
+    IO.popen(@config.sendmail, "w") { |sm| sm.puts content; true }
+    debug "done sending content to #{to_address}"
+  end
+
+  def handle_send_request id, params
+    email = params["email"]
+    raise Error, "invalid email address" unless email =~ /^\S+@\S+$/
+
+    if params["comment-id"]
+      c = @commentset.comments_by_id[params["comment-id"]]
+      raise Error, "can't find comment" unless c
+      @q.push [c, email, true]
+    else
+      e = @entryset.entries_by_id[id]
+      raise Error, "can't find entry" unless e
+      @q.push [e, email, true]
+    end
+  end
+
+  def send_comment comment, to_addr; @q.push [comment, to_addr, false] end
+
+private
+
+  def get_content_for_entry entry, to_address, web_request
+    @entry_template.refresh!
+    debug "creating email from template #{@entry_template.path} and entry #{entry.id}"
+    message_id = message_id_for_entry entry
+    author = entry.author
+    settings = @authors[to_address]
+    ERB.new(@entry_template.content, nil, "%-").result binding
+  end
+
+  def get_content_for_comment comment, to_address, web_request
+    @comment_template.refresh!
+    debug "creating email from template #{@comment_template.path} and comment #{comment.id}"
+    entry = @entryset.entries_by_id[comment.entry_id]
+    message_id = comment.message_id
+
+    reply_to_message_id = if comment.parent_id && (pc = @commentset.comments_by_id[comment.parent_id])
+      pc.message_id
+    else
+      message_id_for_entry entry
+    end
+
+    author = comment.author
+    settings = @authors[to_address]
+    ERB.new(@comment_template.content, nil, "%-").result binding
+  end
+
+  def message_id_for_entry entry; "whisper-post-#{entry.id}@#{@hostname}" end
+end
+end