mail2frontmatter 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 805544b6ae89d533ee1dd0c5d626a3cbf69ad5f2
+  data.tar.gz: eb23c01ec9e523c3b3714d25e0ec30ea06fabf50
+SHA512:
+  metadata.gz: 98b0857153b18118cc9a44e29e456631f7a5bc9b7e2c79de753fc0b900721ad6c257df4b307adf1e03735b67cf16ccc706169950bda7a8bbd3c146c9a8d70562
+  data.tar.gz: 47ca7154d3430195d89ebbe2087eb3ca467274dcf5cf540f4bfe7a247416a8fc2022a4f6f6cdb88287f577a567a853ddd260d6cda2229647e7f9e7a33b598bd9

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mail2frontmatter.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Kunal Shah
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,112 @@
+# Mail2FrontMatter
+
+Email-to-blog parser which creates YAML FrontMatter and saves your attachments.
+
+Designed to be used with either Middleman or Jekyll
+
+## Installation
+
+Install it yourself as:
+
+    $ gem install mail2frontmatter
+
+Or if you do not intend to directly use the executable, you can include it in Gemfile like:
+
+```ruby
+gem 'mail2frontmatter'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+To run in the foreground:
+
+    $ mail2frontmatter
+
+To detach the process pass the ```-d``` option:
+
+    $ mail2frontmatter -d
+
+```mail2frontmatter``` assumes a configuration file will be present at ```./data/mail2frontmatter.yml```
+
+You can override this by passing ```--config=CONFIG```
+
+Furthermore, when run daemonized ```mail2frontmatter``` assumes it can:
+
+1. write its pidfile to ```./tmp/pids/mail2frontmatter.pid```
+2. append its log output to ```./log/mail2frontmatter.log```
+
+You can override each of these settings as well:
+
+To specify a pidfile location set ```--pid=PIDFILE```
+Note: no pidfile is written when run in the foreground
+
+To specify a log file location set ```--log=LOGFILE```
+The default log file when detached is ```./log/mail2frontmatter.log``` (otherwise its ```STDOUT```)
+
+Finally to stop (```SIGTERM```) the detached process call:
+
+    $ mail2frontmatter -k
+
+### Basic Configuration
+
+Your configuration file should by parseable YAML. 
+
+```yaml
+protocol: imap
+receiver: yourblogemail@yourdomain.com
+senders:  yourpersonal@yourdomain.com
+mailman:
+  server: imap.gmail.com
+  port: 993
+  ssl: true
+  username: yourblogemail@yourdomain.com
+  password: yourpassword
+```
+
+As shown the mailman configuration are the exact options you would pass [to that gem](https://github.com/titanous/mailman/blob/master/USER_GUIDE.md).
+
+### Embedded Configuration
+
+Finally, as an alternative to using the executable and configuration you can use the watcher embedded within your own code.
+
+Just instantiate it with an empty hash or with config options, then give it a block. You can directly access the Mailman object here as well. See the example below and look at the gem code for more details.
+
+```ruby
+  require 'mail2frontmatter'
+
+  watcher = Mail2FrontMatter::Watcher.new({}) do |config|
+    config[:mailman] = {
+      server: imap.gmail.com
+      port: 993
+      ssl: true
+      username: youruser@yourdomain.com
+      password: yourpassword
+    }
+
+    @receiver = "foo@bar.com"
+    @senders  = "baz@biz.com"
+
+    ....
+  end
+
+  # run it
+  watcher.run
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/whistlerbrk/mail2frontmatter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+### TODO
+
+* White list sanitization!
+* Mail2FrontMatter::Watcher handles both configuration for the whole shebang as well as Mailman. Should be split
+* Some options intended to be configurable (media directory, etc) are not yet and essentially mean you can only run this from a middleman directory installation atm.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/mail2frontmatter

@@ -0,0 +1,127 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+run_dir = Dir.pwd
+
+# only run time options, not *configuration* options
+stop = false
+config = nil
+options = {
+  background: false,
+  log_file:   nil,
+  pid_file:   File.join(run_dir, 'tmp', 'pids', 'mail2frontmatter.pid')
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: mail2frontmatter [options]"
+
+  opts.on("-k", "--stop", "stop the running process if any") do
+    stop = true
+  end
+
+  opts.on("-cCONFIG", "--config=CONFIG", "path to configuration file, defaults to ./data/mail2frontmatter.yml") do |config_file|
+    require 'yaml'
+    config = YAML.load_file(config_file).symbolize_keys!
+  end
+
+  opts.on("-d", "--detach", "detach process and run in the background") do
+    options[:background] = true
+  end
+
+  opts.on("-lLOGFILE", "--log=LOGFILE", "path to log file, defaults to STDOUT in the foreground or ./log/mail2frontmatter.log when daemonized") do |log_file|
+    options[:log_file] = log_file
+  end
+
+  opts.on("-pPIDFILE", "--pid=PIDFILE", "path to pid file, defaults to ./tmp/pids/mail2frontmatter.pid") do |pid_file|
+    options[:pid_file] = pid_file
+  end
+
+  opts.on("-h", "--help", "Prints this help") do
+    puts opts
+    exit
+  end
+end.parse!
+
+# TRY TO KILL
+if stop
+
+  if !File.exist?(options[:pid_file])
+    puts "no pidfile! are you sure mail2frontmatter is running?"
+    exit 1
+  end
+
+  pid = File.read(options[:pid_file]).to_i
+  puts "stopping (#{pid})... "
+
+  # N.B / TODO - DOESN'T WAIT.. DOESN'T CHECK
+  begin
+    Process.kill('TERM', pid)
+    exit 0
+
+    # FUTURE: library intercepts TERM, 
+    # waits to finish processing if it is
+    # or if kill exits dirty.
+
+  # CATCH STALE PID
+  rescue Errno::ESRCH
+    puts "stale pidfile... cleaning"
+
+  # ALWAYS DELETE PID FILE
+  ensure
+    File.delete(options[:pid_file])
+
+  end
+
+end
+
+# TRY TO START
+require File.join(File.dirname(__FILE__), '..', 'lib', 'mail2frontmatter')
+
+# DAEMONIZE...
+if options[:background]
+
+  # CHECKING ALREADY RUNNING / CHECK STALE
+  if File.exists?(options[:pid_file])
+    pid = File.read(options[:pid_file]).to_i
+
+    begin
+      Process.kill(0, pid)
+      puts "mail2frontmatter is already running (#{pid})"
+      exit 1
+
+    rescue Errno::ESRCH
+      puts "stale pidfile... cleaning"
+      File.delete(options[:pid_file])
+    end
+  end
+
+  puts "daemonizing..."
+
+  # SET LOG FILE INTERNAL
+  watcher = Mail2FrontMatter::Watcher.new(config) do |config|
+    if !options[:log_file]
+      config[:log_file] = File.join(run_dir, 'log', 'mail2frontmatter.log')
+    else
+      config[:log_file] = options[:log_file]
+    end
+  end
+
+  # FORK / DETACH
+  runner = fork do
+    File.write(options[:pid_file], Process.pid)
+    watcher.run
+  end
+
+  Process.detach(runner)
+
+else
+
+  # SET LOG FILE INTERNAL (REDUNDANT)
+  watcher = Mail2FrontMatter::Watcher.new(config) do |config|
+    config[:log_file] = !options[:log_file] ? STDOUT : options[:log_file]
+  end
+
+  # FOREGROUND
+  watcher.run
+end

data/lib/mail2frontmatter/parser.rb

@@ -0,0 +1,72 @@
+
+# Breaks down email
+module Mail2FrontMatter
+  class Parser
+    require 'nokogiri'
+    require 'fileutils'
+
+    attr_accessor :message, :metadata, :body
+
+    ALLOWED_TYPES = {
+      "audio" => "audio",
+      "video" => "videos", 
+      "image" => "images"
+    }
+
+    def initialize(message)
+      @message = message
+      raw_parsed_html = Nokogiri::HTML.parse(@message.html_part.body.raw_source.strip)
+      @body = raw_parsed_html.at("body")
+
+      # remove extraneous nesting
+      while(@body.children.count == 1 && @body.children.first.name == "div") do
+        @body = @body.children.first
+      end
+
+      attachments = {}
+
+      @message.attachments.each do |attachment|
+        if Parser::ALLOWED_TYPES.keys.include?(attachment.main_type)
+
+          # save attachments
+          media_directory = File.join(Dir.pwd, 'source', Parser::ALLOWED_TYPES[attachment.main_type])
+          FileUtils.mkdir_p(media_directory)
+
+          filepath = File.join(media_directory, attachment.filename) 
+
+          # save attachment
+          File.open(filepath, "w+b", 0644) {|f| f.write attachment.body.decoded}
+
+          # retain metadata
+          attachments[attachment.cid] = {
+            maintype: attachment.main_type,
+            mimetype: attachment.mime_type,
+            filename: attachment.filename,
+            filepath: filepath
+          }
+
+        # file type not allowed
+        else
+          # remove cooresponding node from html
+          @body.xpath("//*[@src='cid:#{attachment.content_id}']").remove
+
+        end
+      end
+
+      # convert body immediately to a string, why?
+      # Plugins / pre-processors may wish to manipulate the body
+      # however Nokogiri is strict and won't allow template tags
+      # for obvious good reasons
+      @body = @body.inner_html
+
+      @metadata = {
+        from:        message[:to].value,
+        to:          message[:from].value,
+        received:    message.date,
+        subject:     message.subject,
+        attachments: attachments
+      }
+
+    end
+  end
+end

data/lib/mail2frontmatter/preprocessor.rb

@@ -0,0 +1,46 @@
+
+# Pre-processes blog data, allows hooks
+# 
+# 
+# Build a Pre-Processor that accepts two arguments
+# metadata, a symbolized hash of metadata extracted from the email
+# and body, a Nokogiri parsed representation of the email
+# 
+# modify as necessary and as atomically as possible and return two arguments
+# 
+# example:
+# 
+# module Mail2FrontMatter
+#   class MyProcessor < PreProcessor
+#     def self.run(metadata, body)
+#       metadata[:some_field] = some_transformation_of(metadata[:some_field])
+#       return metadata, body
+#     end
+#   end
+# end
+
+module Mail2FrontMatter
+  class PreProcessor
+    require 'set'
+
+    class InvalidProcessor < StandardError ; end
+
+    @@processors = Set.new
+
+    def self.register(options = {})
+      raise InvalidProcessor, "run method not defined on #{self}" if !self.respond_to?(:run)
+      @options = options
+
+      @@processors << self
+    end
+
+    def self.process(metadata, body)
+      @@processors.each do |processor|
+        metadata, body = processor.run(metadata, body)
+      end
+
+      return metadata, body
+    end
+
+  end
+end

data/lib/mail2frontmatter/version.rb

@@ -0,0 +1,3 @@
+module Mail2FrontMatter
+  VERSION = "0.0.1"
+end

data/lib/mail2frontmatter/watcher.rb

@@ -0,0 +1,79 @@
+
+# configures and runs Mailman
+module Mail2FrontMatter
+  require 'mailman'
+  require 'byebug'
+  require 'active_support/inflector'
+
+  class Watcher
+
+    attr_accessor :logger
+
+    def initialize(config = nil, &block)
+      # load config from file
+      if config.is_a?(String)
+        config = YAML.load_file(config).deep_symbolize_keys!
+
+      # load config from file at default data/mail2frontmatter.yml relative from run directory
+      elsif config.is_a?(NilClass)
+
+        default_config = File.join(Dir.pwd, 'data', 'mail2frontmatter.yml')
+
+        if File.exist?(default_config)
+          config = YAML.load_file(default_config).deep_symbolize_keys!
+        else
+          raise LoadError, 'no configuration given or found at ./data/mail2frontmatter.yml'
+        end
+
+      elsif !config.is_a?(Hash)
+        raise ArgumentError, 'not a valid configuration type'
+      end
+
+      yield(config) if block_given?
+
+      preprocessors = config.delete(:preprocessors) || []
+      preprocessors.each do |processor|
+
+        begin
+          require "mail2frontmatter/#{processor[:key]}"
+        rescue LoadError => e
+          puts "could not require specified preprocessor '#{processor[:key]}.rb', no such file in load path. Check your configuration and try again \n\n"
+          raise e
+        end
+
+        klass = "Mail2FrontMatter::#{processor[:key].underscore.classify}".constantize.register(processor[:options])
+      end
+
+      mail_protocol = config.delete(:protocol) || :imap
+      poll_interval = config.delete(:interval) || 60
+
+      @receiver = config.delete(:receiver)
+      @senders  = config.delete(:senders)
+      @logger   = Logger.new(config.delete(:log_file))
+
+      Mailman.config.poll_interval = poll_interval
+      Mailman.config.ignore_stdin = true
+
+      Mailman.config.send("#{mail_protocol}=", config[:mailman])
+      Mailman.config.logger = @logger
+    end
+
+    def run
+      Mailman::Application.run do
+        from(@senders).to(@receiver) do
+          logger = Mailman.config.logger
+
+          logger.info('parsing message...')
+          parser = Mail2FrontMatter::Parser.new(message)
+
+          logger.info('processing body and attachments...')
+          metadata, body = Mail2FrontMatter::PreProcessor.process(parser.metadata, parser.body)
+
+          logger.info('saving processed post...')
+          Mail2FrontMatter::Writer.write(metadata, body)
+        end
+      end
+    end
+
+  end
+end

data/lib/mail2frontmatter/writer.rb

@@ -0,0 +1,41 @@
+
+# Writes YAML FrontMatter & Article Content in ERB
+# 
+# TODO How do I support other templating solutions here?
+#      Same plugin style as processors?
+#      Is processor and writer doing the same thing?
+#       --- probably just let anyone specify the extension.. keep it simple
+# 
+module Mail2FrontMatter
+  class Writer
+
+    require 'yaml'
+    require 'active_support/inflector'
+
+    def self.write(metadata, body)
+
+      # TODO FIXME!
+      # 
+      # this is supposed to be configurable!
+      # get this value from a module variable!
+      # 
+      data_directory = File.join(Dir.pwd, 'source', 'blog')
+
+      # MAPPINGS!
+      # 
+      # Play nice with programs which will read this data
+      # And set sensible defaults as fall throughs
+
+      # if there is no title set, borrow the subject lines
+      metadata[:title] ||= metadata[:subject]
+
+      # make a sensible standard blog filename unless one is given
+      metadata[:filename] ||= [metadata[:received].strftime("%Y-%m-%d"), '-', metadata[:subject].parameterize, '.html.erb'].join
+
+      data = metadata.to_yaml + "---\n" + body
+
+      File.write(File.join(data_directory, metadata[:filename]), data)
+    end
+
+  end
+end

data/lib/mail2frontmatter.rb

@@ -0,0 +1,5 @@
+require 'mail2frontmatter/version'
+require 'mail2frontmatter/watcher'
+require 'mail2frontmatter/parser'
+require 'mail2frontmatter/writer'
+require 'mail2frontmatter/preprocessor'

data/mail2frontmatter.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mail2frontmatter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mail2frontmatter"
+  spec.version       = Mail2FrontMatter::VERSION
+  spec.authors       = ["Kunal Shah"]
+  spec.email         = ["me@kunalashah.com"]
+  spec.summary       = %q{Email-to-blog parser which creates YAML FrontMatter}
+  spec.description   = spec.summary + %q{. Uses Mailman to poll an account. }
+  spec.homepage      = "https://github.com/whistlerbrk/Mail2FrontMatter"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "mailman"
+  spec.add_dependency "activesupport"
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: mail2frontmatter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Kunal Shah
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mailman
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 'Email-to-blog parser which creates YAML FrontMatter. Uses Mailman to
+  poll an account. '
+email:
+- me@kunalashah.com
+executables:
+- mail2frontmatter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/mail2frontmatter
+- lib/mail2frontmatter.rb
+- lib/mail2frontmatter/parser.rb
+- lib/mail2frontmatter/preprocessor.rb
+- lib/mail2frontmatter/version.rb
+- lib/mail2frontmatter/watcher.rb
+- lib/mail2frontmatter/writer.rb
+- mail2frontmatter.gemspec
+homepage: https://github.com/whistlerbrk/Mail2FrontMatter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Email-to-blog parser which creates YAML FrontMatter
+test_files: []